scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-05 08:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 136

Merge remote-tracking branch 'origin/main' into litellm_bedrock_messages_disconnect_billing

Browse source 
# Conflicts:
#	basedpyright-code-budget.json
#	litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
#	tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py
#	type-discipline-budget.json
This commit is contained in:
nuernber 2026-08-31 08:58:37 -07:00
commit 95a2586228
4984 changed files with 414786 additions and 108737 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

320
.circleci/config.yml
View file

@ -430,7 +430,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -504,7 +504,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -631,7 +631,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -651,126 +651,6 @@ jobs:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
proxy_behavior_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy management behavior tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_behavior \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
proxy_security_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy security tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_security_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
schema_migration_check:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
# An empty database; the test applies every committed migration itself.
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Check schema.prisma is in sync with committed migrations
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_migration_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- *python312_image
@ -858,7 +738,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -985,7 +865,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
@ -1025,12 +905,12 @@ jobs:
name: Run tests
command: |
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
TEST_FILES=$(circleci tests glob "tests/agent_tests/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1074,7 +954,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
@ -1120,7 +1000,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--retries 3 --retry-delay 5"
@ -1211,7 +1091,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1255,7 +1135,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1274,40 +1154,6 @@ jobs:
paths:
- search_coverage.xml
- search_coverage
litellm_mapped_enterprise_tests:
docker:
- *python312_image
working_directory: ~/project
resource_class: large
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- setup_litellm_enterprise_pip
- run:
name: Run enterprise tests
command: |
uv run --no-sync python -m prisma generate
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-enterprise.xml \
--durations=10 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
batches_testing:
docker:
- *python312_image
@ -1333,7 +1179,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1377,7 +1223,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1422,7 +1268,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1501,7 +1347,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
-n 4 \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1546,7 +1392,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1599,7 +1445,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1"
@ -2575,45 +2421,6 @@ jobs:
- wait_for_service:
url: http://localhost:4000
timeout: "300"
# Add Ruby installation and testing before the existing Node.js and Python tests
- run:
name: Install Ruby and Bundler
command: |
# Clone RVM at pinned tag and verify the commit SHA matches the
# published tag before running its install script.
RVM_VERSION="1.29.12"
RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81"
git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm
RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)"
if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then
echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2
exit 1
fi
# Import RVM signing keys (used by `rvm install` to verify Ruby tarballs)
gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
# Install RVM from the verified checkout. The install script
# sources `scripts/functions/installer` using paths relative to
# its own working directory, so it must be run from /tmp/rvm.
(cd /tmp/rvm && ./install --path "$HOME/.rvm")
source "$HOME/.rvm/scripts/rvm"
# Install Ruby 3.2.2 (RVM verifies the tarball PGP signature)
rvm install 3.2.2
rvm use 3.2.2 --default
# Install latest Bundler
gem install bundler
- run:
name: Run Ruby tests
command: |
source $HOME/.rvm/scripts/rvm
cd tests/pass_through_tests/ruby_passthrough_tests
bundle install
bundle exec rspec
no_output_timeout: 30m
# Install Node.js directly from nodejs.org with SHA256 verification,
# instead of piping NodeSource's setup_24.x apt-repo installer into
# sudo bash (which runs a mutable upstream script unattended).
@ -2744,84 +2551,6 @@ jobs:
file: ./coverage.xml
flags: circleci
ui_build:
docker:
- image: cimg/node:24.19@sha256:8966565f07189a67d64d6808a2b127f31dafae566508e3547f55640e1070bfad
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
resource_class: medium+
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- restore_cache:
keys:
- ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- ui-build-deps-v1-
- restore_cache:
keys:
- ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- ui-nextjs-cache-v1-
- run:
name: Install dependencies
command: |
cd ui/litellm-dashboard
npm ci
- save_cache:
key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- run:
name: Build UI
command: |
cd ui/litellm-dashboard
source ./build_ui.sh
- save_cache:
key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
paths:
- ui/litellm-dashboard/.next/cache
- persist_to_workspace:
root: .
paths:
- litellm/proxy/_experimental/out
ui_unit_tests:
docker:
- image: cimg/node:24.19@sha256:8966565f07189a67d64d6808a2b127f31dafae566508e3547f55640e1070bfad
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- skip_if_unrelated_changes:
category: client
- setup_google_dns
- restore_cache:
keys:
- ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
- ui-unit-deps-v1-
- run:
name: Install dependencies
command: |
cd ui/litellm-dashboard
npm ci
- save_cache:
key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
paths:
- ui/litellm-dashboard/node_modules
- run:
name: Run UI unit tests (Vitest)
command: |
cd ui/litellm-dashboard
CI=true npm run test -- --run \
--pool forks --poolOptions.forks.maxForks=6
e2e_ui_testing:
docker:
- image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
@ -3181,20 +2910,8 @@ workflows:
filters: *main_branches
- litellm_router_unit_testing:
filters: *main_branches
- ui_build:
filters: *main_branches
- ui_unit_tests:
requires:
- ui_build
filters: *main_branches
- auth_ui_unit_tests:
filters: *main_branches
- proxy_behavior_tests:
filters: *main_branches
- proxy_security_tests:
filters: *main_branches
- schema_migration_check:
filters: *main_branches
- build_docker_database_image:
filters: *main_branches
- e2e_ui_testing:
@ -3251,8 +2968,6 @@ workflows:
filters: *main_branches
- search_testing:
filters: *main_branches
- litellm_mapped_enterprise_tests:
filters: *main_branches
- batches_testing:
filters: *main_branches
- litellm_utils_testing:
@ -3275,7 +2990,6 @@ workflows:
- guardrails_testing
- ocr_testing
- search_testing
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing

7
.circleci/scripts/classify_changes.sh
View file

@ -1,15 +1,17 @@
#!/usr/bin/env bash
set -uo pipefail
category="${1:?usage: classify_changes.sh <backend|client>}"
category="${1:?usage: classify_changes.sh <backend|client|ui>}"
has_client=false
has_backend=false
has_ci=false
while IFS= read -r file || [ -n "$file" ]; do
[ -n "$file" ] || continue
case "$file" in
ui/* | tests/e2e/ui/*) has_client=true ;;
docs/* | *.md | *.mdx) : ;;
.github/* | .circleci/*) has_ci=true; has_backend=true ;;
*) has_backend=true ;;
esac
done
@ -21,6 +23,9 @@ case "$category" in
client)
{ [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip
;;
ui)
{ [ "$has_client" = true ] || [ "$has_ci" = true ]; } && echo run || echo skip
;;
*)
echo run
;;

11
.github/CODEOWNERS vendored
View file

@ -1,3 +1,10 @@
/ui/ @yuneng-jiang @ryan-crabbe-berri
/litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri
/ui/ @yuneng-berri @ryan-crabbe-berri
/litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
/ui/Dockerfile
/ui/nginx.conf
/ui/litellm-dashboard/src/lib/http/schema.d.ts
/ui/litellm-dashboard/tsconfig.tsbuildinfo
/model_prices_and_context_window.json @mateo-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri
/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri
/.github/CODEOWNERS @yuneng-berri

31
.github/actions/cache-cargo-build/action.yml vendored Normal file
View file

@ -0,0 +1,31 @@
name: "Cache the Rust build"
description: >-
Cache the Cargo registry and target directory the root package's build needs,
so only the first job on a given Cargo.lock compiles the bridge from scratch.
litellm builds through maturin, which compiles litellm-rust/crates/python-bridge
in release mode before it can produce a wheel. `uv sync` therefore pays a full
build in every job that installs the workspace: measured at 2m40s per unit shard
on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught
it, because the uv cache holds wheels uv downloads rather than wheels it builds,
and a path dependency whose source moves every commit could never hit that cache
anyway. Cargo rebuilds only what changed when its target directory survives, so a
warm job pays for the bridge crate alone.
The key namespace is separate from test-rust.yml's. Both cache the same directory,
but that workflow fills it with debug and clippy artifacts, which a release build
cannot reuse, and a shared key would let whichever ran first deny the other a save.
runs:
using: composite
steps:
- name: Restore the Cargo registry and target directory
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cargo/registry
~/.cargo/git
litellm-rust/target
key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-release-

48
.github/actions/detect-backend-changes/action.yml vendored
View file

@ -1,48 +0,0 @@
name: "Detect backend-relevant changes"
description: >-
Classify the pull request's changed files with .circleci/scripts/classify_changes.sh
and expose decision=run|skip. decision=skip means only ui/**, **.md or **.mdx files
changed, so callers can short-circuit expensive steps while the job still completes
successfully and satisfies its required status check. The decision defaults to run for
any non pull_request event or whenever the changed set cannot be resolved, so tests are
never skipped when the classification is uncertain.
outputs:
decision:
description: "run when backend-relevant files changed, otherwise skip"
value: ${{ steps.classify.outputs.decision }}
runs:
using: composite
steps:
- id: classify
shell: bash
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
set -uo pipefail
if [ -z "${BASE_SHA:-}" ]; then
echo "detect-backend-changes: not a pull_request event; running job"
echo "decision=run" >> "${GITHUB_OUTPUT}"
exit 0
fi
if ! git fetch --no-tags --depth=1 origin "${BASE_SHA}" >/dev/null 2>&1; then
echo "detect-backend-changes: could not fetch base ${BASE_SHA}; running job"
echo "decision=run" >> "${GITHUB_OUTPUT}"
exit 0
fi
changed="$(git diff --name-only "${BASE_SHA}" HEAD 2>/dev/null)" || {
echo "detect-backend-changes: git diff failed; running job"
echo "decision=run" >> "${GITHUB_OUTPUT}"
exit 0
}
if [ -z "${changed}" ]; then
echo "detect-backend-changes: no changed files vs ${BASE_SHA}; skipping job"
echo "decision=skip" >> "${GITHUB_OUTPUT}"
exit 0
fi
echo "detect-backend-changes: changed files vs ${BASE_SHA}:"
printf '%s\n' "${changed}" | sed 's/^/ /'
decision="$(printf '%s\n' "${changed}" | bash .circleci/scripts/classify_changes.sh backend)" || decision="run"
echo "detect-backend-changes: decision=${decision}"
echo "decision=${decision}" >> "${GITHUB_OUTPUT}"

41
.github/actions/detect-changes/action.yml vendored Normal file
View file

@ -0,0 +1,41 @@
name: "Detect relevant changes"
description: >-
Classify the pull request's changed files with .circleci/scripts/classify_changes.sh
and expose decision=run|skip for one category. backend means anything outside ui/,
docs/ and markdown; ui means the dashboard sources alone. decision=skip lets callers
short-circuit expensive steps while the job still completes successfully and satisfies
its required status check, which a paths: filter cannot do because a workflow that
never starts never reports. The file list comes from the pull request itself rather
than from a git diff, because the checked-out merge ref is recomputed as the base
branch advances and would otherwise attribute the base branch's own commits to the
pull request. The decision defaults to run for any non pull_request event or whenever
the changed set cannot be resolved, so jobs are never skipped when the classification
is uncertain.
inputs:
category:
description: "Which classification to apply: backend, client or ui"
required: false
default: backend
github-token:
description: "Token used to list the pull request's files; needs pull-requests: read"
required: false
default: ${{ github.token }}
outputs:
decision:
description: "run when category-relevant files changed, otherwise skip"
value: ${{ steps.classify.outputs.decision }}
runs:
using: composite
steps:
- id: classify
shell: bash
env:
GH_TOKEN: ${{ inputs.github-token }}
CATEGORY: ${{ inputs.category }}
REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
CHANGED_FILE_COUNT: ${{ github.event.pull_request.changed_files }}
run: bash "${GITHUB_ACTION_PATH}/../../scripts/detect_changes.sh"

105
.github/ci-coverage-allowlist.yml vendored
View file

@ -4,6 +4,22 @@ description: >-
by a job nor listed here, so every entry below is a decision on the record.
test_paths:
- reason: >-
What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
split that porting tests/local_testing off CircleCI will force, not a job that is red by
construction
paths:
- tests/local_testing/test_caching.py
- tests/local_testing/test_disk_cache_unit_tests.py
- tests/local_testing/test_gcs_cache_unit_tests.py
- reason: >-
The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
from a pull request; it needs a live gateway and provider credentials no PR job holds
@ -21,72 +37,14 @@ test_paths:
- tests/documentation_tests/test_requests_lib_usage.py
- tests/documentation_tests/test_standard_logging_payload.py
- reason: >-
Sibling files here are executed by name from the code-quality workflow; this one is referenced
by no job
Named like a test but shaped like a benchmark: it fetches live image URLs, times aiohttp
against httpx, prints the ratio, and asserts nothing, so pytest cannot collect it (its
functions take arguments, not fixtures) and running it beside its siblings in the
code-quality workflow would add a network dependency for a number nothing reads. Exempt
as a script rather than as an unresolved gap; revisit by deleting it once the aiohttp
choice it informed is settled
paths:
- tests/code_coverage_tests/test_aio_http_image_conversion.py
- reason: >-
A second mirror of the package tree living beside tests/test_litellm, which is the mirror the
repo convention names; only test_no_hardcoded_secrets.py is invoked, from the linting
workflow, and whether this directory should exist at all is unresolved
paths:
- tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_headers.py
- tests/litellm/a2a_protocol/providers/pydantic_ai_agents/test_pydantic_ai_agent_transformation.py
- tests/litellm/integrations/helicone/test_helicone_gemini.py
- tests/litellm/litellm_core_utils/test_json_schema_validation.py
- tests/litellm/llms/anthropic/test_anthropic_reasoning_effort.py
- tests/litellm/llms/anthropic/test_anthropic_schema_filter.py
- tests/litellm/llms/azure/test_azure_embedding.py
- tests/litellm/llms/bedrock/embed/test_embedding.py
- tests/litellm/llms/bedrock/test_nova_imported_models.py
- tests/litellm/llms/deepseek/chat/test_deepseek_chat_transformation.py
- tests/litellm/llms/gradient_ai/chat/test_gradient_ai_chat_transformation.py
- tests/litellm/llms/oci/chat/test_oci_chat_transformation.py
- tests/litellm/llms/openai_like/test_abliteration_provider.py
- tests/litellm/llms/openai_like/test_assemblyai_provider.py
- tests/litellm/llms/openai_like/test_empiriolabs_provider.py
- tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py
- tests/litellm/llms/vertex_ai/gemini/test_transformation.py
- tests/litellm/llms/vertex_ai/text_to_speech/test_transformation.py
- tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
- tests/litellm/proxy/agent_endpoints/test_agent_rbac.py
- tests/litellm/proxy/common_utils/test_rbac_utils.py
- tests/litellm/proxy/management_endpoints/test_common_utils.py
- tests/litellm/proxy/management_endpoints/test_cost_estimate_endpoint.py
- tests/litellm/proxy/test_claude_code_marketplace.py
- tests/litellm/proxy/test_init_litellm_callbacks.py
- tests/litellm/proxy/test_prisma_engine_watchdog.py
- tests/litellm/proxy/vector_store_endpoints/test_vector_store_rbac.py
- tests/litellm/test_bedrock_extended_beta_models.py
- tests/litellm/test_bedrock_nemotron_super.py
- tests/litellm/test_proxy_auth.py
- tests/litellm/test_router_retry_backoff_headers.py
- tests/litellm/test_sambanova_model_metadata.py
- tests/litellm/test_stream_chunk_builder_images.py
- reason: >-
Legacy proxy suite superseded by the proxy shards; no job invokes it and whether it still
describes supported behaviour is unresolved
paths:
- tests/old_proxy_tests/tests/test_anthropic_context_caching.py
- tests/old_proxy_tests/tests/test_anthropic_sdk.py
- tests/old_proxy_tests/tests/test_async.py
- tests/old_proxy_tests/tests/test_gemini_context_caching.py
- tests/old_proxy_tests/tests/test_langchain_embedding.py
- tests/old_proxy_tests/tests/test_langchain_request.py
- tests/old_proxy_tests/tests/test_llamaindex.py
- tests/old_proxy_tests/tests/test_mistral_sdk.py
- tests/old_proxy_tests/tests/test_openai_embedding.py
- tests/old_proxy_tests/tests/test_openai_exception_request.py
- tests/old_proxy_tests/tests/test_openai_request.py
- tests/old_proxy_tests/tests/test_openai_request_with_traceparent.py
- tests/old_proxy_tests/tests/test_openai_simple_embedding.py
- tests/old_proxy_tests/tests/test_openai_tts_request.py
- tests/old_proxy_tests/tests/test_pass_through_langfuse.py
- tests/old_proxy_tests/tests/test_q.py
- tests/old_proxy_tests/tests/test_simple_traceparent_openai.py
- tests/old_proxy_tests/tests/test_vertex_sdk_forward_headers.py
- tests/old_proxy_tests/tests/test_vtx_embedding.py
- tests/old_proxy_tests/tests/test_vtx_sdk_embedding.py
- reason: >-
No job invokes this suite and its files mix pure transformation tests with ones driving live
vendor vector stores, so assigning them needs a per-file decision
@ -116,6 +74,14 @@ test_paths:
- tests/load_tests/test_otel_load_test.py
- tests/load_tests/test_vertex_embeddings_load_test.py
- tests/load_tests/test_vertex_load_tests.py
- reason: >-
A local-only agent rig: test_a2a_completion_bridge.py needs a LangGraph server on
localhost:2024 and test_a2a.py drives a live A2A endpoint, so neither can run in a
pull request job. Until 2026-08-20 the CircleCI agent job hid them behind a grep -v
that this census could not see; the glob now excludes them structurally and this entry
is the decision on the record. Revisit when the A2A bridge gets a recorded-wire fixture
paths:
- tests/agent_tests/local_only_agent_tests
- reason: >-
Third-party integration tests that skip themselves without OCI configuration or sandbox
credentials, neither of which a pull request job holds
@ -123,17 +89,6 @@ test_paths:
- tests/integration/sandbox/test_e2b_sandbox.py
- tests/integration/test_oci_integration.py
- tests/integration/test_oci_proxy_integration.py
- reason: >-
Two prompt-factory tests sitting at the top level of tests/ instead of under the
tests/test_litellm mirror the shards enumerate; they need moving rather than a shard entry
paths:
- tests/litellm_core_utils/test_anthropic_dedup_factory.py
- tests/litellm_core_utils/test_bedrock_converse_dedup_factory.py
- reason: >-
A unit test for the proxy-extras package that no job invokes, while the package's other tests
live under tests/proxy_migration_tests
paths:
- tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
dockerfiles:
- reason: >-

5
.github/mutmut-coverage.rc vendored Normal file
View file

@ -0,0 +1,5 @@
# mutmut's gather_coverage() looks covered lines up by absolute path, so the
# repo's `relative_files = true` makes every lookup miss and mutmut generates
# zero mutants. Point COVERAGE_RCFILE here for mutation runs only.
[run]
relative_files = false

62
.github/pull_request_template.md vendored
View file

@ -1,7 +1,10 @@
<!-- The whole description's target audience is humans, not AI agents: write it in plain, simple,
everyday engineering language, extremely parsable and readable at a glance. This goes double for
the TLDR, User Flow, and Caveats sections -->
## TLDR
<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max -->
Problem this solves:
@ -53,7 +56,8 @@ After: the same request comes back with real token counts, so the dashboard show
**Please complete all items before asking a LiteLLM maintainer to review your PR**
- [ ] I have added meaningful tests
- [ ] My PR passes all CI/CD checks (e.g., lint, format, unit tests)
- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/test_litellm/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
- [ ] My PR passes all required CI/CD checks (e.g., lint, schema.d.ts sync check, etc.)
- [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
- [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes)
@ -64,12 +68,36 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
## Screenshots / Proof of Fix
<!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
The proof must be completely e2e with no mocks, using, for example, actual LLM calls costing real $. `pytest` commands are not enough
For bug fixes: show reproduction before the fix and passing behavior after
Include the commit hash each proof was captured at, for both the before and the after runs
If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every single one of them, not just one
For new features: show the feature working end-to-end
For UI changes: include before/after screenshots -->
The proof must be completely e2e with no mocks, using actual LLM calls costing real $$$ if applicable. `pytest` commands are not enough
Show ONLY the latest run: capture Before at the merge base and After at the PR's current tip, and when new commits change behavior, replace this whole section with the fresh run instead of stacking it on top of older ones. The run must be up to date. As soon as a new commit is made and it makes this PR description's after sha stale (it's no longer tip of PR), you must re-run the QA
Structure the section exactly as below: Before and After one heading level below this section, each naming the commit hash it was captured at, one lower-level heading per case inside each, the same case names in the same order on both sides, and numbered steps (command, observed output) under every case, never loose prose; shared setup (config, payloads) goes above Before, and with a single case, drop the case headings and number the steps directly
### Before (<hash>)
#### <case 1>
1. ...
2. ...
#### <case 2>
1. ...
### After (<hash>)
#### <case 1>
1. ...
2. ...
#### <case 2>
1. ...
For bug fixes: Before shows the reproduction, After shows the same steps passing
For new features: Before shows the capability missing, After shows it working end-to-end
If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), make each endpoint its own case, not just one
For UI changes: before/after screenshots under the same headings -->
## Type
@ -85,8 +113,20 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
## Caveats (if any)
<!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
<!-- Group caveats under severity subheadings (### Severe, ### High, ### Medium, ### Low), with
short bullet points inside each, just like the TLDR: one line per bullet, roughly 10 words max
Call out known limitations, follow-up work, or anything a reviewer should watch out for
Include only the tiers that have caveats; drop the empty ones
- Severe: inherent to what the PR deliberately ships, there even when the code works as intended:
it can degrade or take down a running deployment (e.g. a slow or table-locking boot migration),
rewrite data by design, break an existing workflow on purpose, or change auth behavior. An
operator must plan around it before rollout
- High: an unintended hole: a correctness, security, data-loss, or backward-compatibility bug,
unsafe to ship as is
- Medium: a real gap someone can hit, but with a workaround or a narrow blast radius
- Low: anything else worth noting: naming, cleanup, an edge case nobody hits
Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a
human reader
Leave this section empty if there are none -->
## QA runbook
@ -109,6 +149,6 @@ Example checklists:
- [ ] Sanity check: this test makes sense to add and is not hand-wavey (e.g., assert actual expected spend instead of just spend > 0) or potentially flaky
-->
### Final Attestation
## Final Attestation
- [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR

295
.github/scripts/assert_ci_coverage.py vendored
View file

@ -1,10 +1,14 @@
from __future__ import annotations
import ast
import operator
import pathlib
import re
import sys
from collections.abc import Iterable, Mapping, Sequence
import warnings
from collections.abc import Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from typing import Final
import yaml
@ -25,6 +29,15 @@ DOCKERFILE_TOKEN_RE = re.compile(r"[A-Za-z0-9_./-]*Dockerfile[A-Za-z0-9_.-]*")
COMMENT_RE = re.compile(r"^\s*#.*$", re.MULTILINE)
GLOB_CHARS = frozenset("*?")
# Trees whose jobs are sharded with no catch-all bucket, so every child that holds
# tests has to be named by some shard or it runs nowhere. A child listed here is
# itself decomposed one level deeper and is checked through its own entry.
SHARDED_ROOTS: tuple[str, ...] = (
"tests/proxy_unit_tests",
"tests/test_litellm",
"tests/test_litellm/proxy",
)
@dataclass(frozen=True, slots=True)
class AllowEntry:
@ -44,6 +57,14 @@ class Allowlist:
return any(relative_path == path for entry in self.dockerfiles for path in entry.paths)
@dataclass(frozen=True, slots=True)
class Section:
name: str
entries: tuple[AllowEntry, ...]
candidates: tuple[str, ...]
matches: Callable[[str, str], bool]
@dataclass(frozen=True, slots=True)
class Scalar:
key: str
@ -107,20 +128,34 @@ def _built_dockerfile_tokens(scalars: Iterable[Scalar]) -> frozenset[str]:
)
def _glob_to_regex(token: str) -> re.Pattern[str]:
parts = re.split(r"(\*\*/|\*\*|\*|\?)", token)
def _glob_to_regex(token: str, *, subtree: bool) -> re.Pattern[str]:
parts = re.split(r"(\*\*/|\*\*|\*|\?|\[[^\]]*\])", token)
translated = "".join(
{"**/": r"(?:.*/)?", "**": r".*", "*": r"[^/]*", "?": r"[^/]"}.get(part, re.escape(part)) for part in parts
{"**/": r"(?:.*/)?", "**": r".*", "*": r"[^/]*", "?": r"[^/]"}.get(part)
or (part if part.startswith("[") and part.endswith("]") else re.escape(part))
for part in parts
)
return re.compile(rf"{translated}(?:/.*)?$")
return re.compile(rf"{translated}(?:/.*)?$" if subtree else rf"{translated}$")
def _token_covers(token: str, relative_path: str) -> bool:
if GLOB_CHARS & set(token):
return _glob_to_regex(token).match(relative_path) is not None
return _glob_to_regex(token, subtree=True).match(relative_path) is not None
return relative_path == token or relative_path.startswith(f"{token}/")
def _token_names(token: str, relative_path: str) -> bool:
"""Whether the token names this path itself, rather than merely containing it.
A sharded tree has no catch-all bucket, so the ancestor token the census is happy
with (`tests/x` standing in for everything below it) is exactly what would let a
newly added child ride along without a shard.
"""
if GLOB_CHARS & set(token):
return _glob_to_regex(token, subtree=False).match(relative_path) is not None
return token == relative_path
def _test_files() -> tuple[str, ...]:
return tuple(
sorted(
@ -166,6 +201,190 @@ def _describe(paths: tuple[str, ...]) -> str:
return f"{len(paths)} test file(s) invoked by no job: {names}{suffix}"
GLOB_CALL_RE = re.compile(r'circleci tests glob "([^"]+)"')
KEYWORD_RE = re.compile(r"-k\s+\\?[\"']([^\"'\\]+)")
@dataclass(frozen=True, slots=True)
class Slice:
"""One job's selection: the files it globs, narrowed by its `-k` expression."""
job: str
globs: tuple[str, ...]
named: frozenset[str]
required: tuple[str, ...]
excluded: tuple[str, ...]
understood: bool
def claims(self, relative_path: str, inner_names: frozenset[str]) -> bool:
"""Whether this job runs any test in the file.
The question is deliberately per-file, not per-test. An excluded term is only
honoured when it appears in the path, because that is the case where it takes
the whole module with it; a term matching one function inside drops that test
and leaves the file claimed. Losing a whole file is the failure worth a gate,
and answering per-test would mean a baseline of test ids that churns on every
rename.
"""
if relative_path in self.named:
return True
if not any(_token_covers(glob, relative_path) for glob in self.globs):
return False
if not self.understood:
return True # a `-k` this parser cannot model is assumed to claim everything
if any(term.lower() in relative_path.lower() for term in self.excluded):
return False
return not self.required or any(
term.lower() in name.lower() for term in self.required for name in inner_names
)
def _strings(node: object) -> Iterable[str]:
if isinstance(node, str):
yield node
elif isinstance(node, dict):
for value in node.values():
yield from _strings(value)
elif isinstance(node, list):
for value in node:
yield from _strings(value)
def _keyword_terms(
expressions: Sequence[str], *, attributable: bool = True
) -> tuple[tuple[str, ...], tuple[str, ...], bool]:
"""A `-k` expression as (required, excluded, understood).
Only flat `and` chains of bare terms are modelled. Anything with `or`, parentheses
or negation of a group is left unmodelled, and its job is then treated as claiming
every file it globs, so an unparsed selector can never raise a false alarm.
`attributable` is False when a job runs several pytest commands, since a selector
read out of the job's text cannot then be tied to the glob it belongs to, and
pairing one command's exclusion with another's glob would invent a gap.
"""
terms: Final = tuple(part.strip() for expression in expressions for part in expression.split(" and "))
if not attributable and terms:
return (), (), False
if any(("or " in term) or ("(" in term) or (term.startswith("not ") and " " in term[4:]) for term in terms):
return (), (), False
return (
tuple(term for term in terms if term and not term.startswith("not ")),
tuple(term[4:].strip() for term in terms if term.startswith("not ")),
True,
)
def _slices() -> tuple[Slice, ...]:
if not CIRCLECI_CONFIG.exists():
return ()
jobs: Final = yaml.safe_load(CIRCLECI_CONFIG.read_text()).get("jobs", {})
return tuple(
Slice(job=job, globs=globs, named=named, required=required, excluded=excluded, understood=understood)
for job, body in jobs.items()
for text in ("\n".join(_strings(body)),)
if "pytest" in text
for globs in (tuple(GLOB_CALL_RE.findall(text)),)
for named in (frozenset(TEST_TOKEN_RE.findall(text)) & frozenset(_test_files()),)
for required, excluded, understood in (
_keyword_terms(tuple(KEYWORD_RE.findall(text)), attributable=len(globs) < 2),
)
if globs or named
)
def _matchable_names(relative_path: str) -> frozenset[str]:
"""Every name a `-k` term can match for this file: its path, plus the names inside it.
pytest matches a keyword against an item's own name and each of its parents', so a
positive term hits a file when it appears in the path or in a class or function name.
"""
try:
with warnings.catch_warnings():
warnings.simplefilter("ignore") # test files carry stray escapes; their names still parse
tree: Final = ast.parse((REPO_ROOT / relative_path).read_text())
except (OSError, SyntaxError):
return frozenset({relative_path})
return frozenset({relative_path}) | frozenset(
node.name
for node in ast.walk(tree)
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
)
def _workflow_named_tokens() -> frozenset[str]:
"""Test tokens a GitHub Actions job names directly.
A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a
workflow names it, so the slice check has to credit those the same way the census does.
"""
return _invoked_test_tokens(
scalar
for path in _config_files()
if path != CIRCLECI_CONFIG
for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
)
def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
slices: Final = _slices()
named_by_workflow: Final = _workflow_named_tokens()
globbed: Final = tuple(
path
for path in _test_files()
if any(_token_covers(glob, path) for slice_ in slices for glob in slice_.globs)
)
return tuple(
Finding(
subject=path,
detail="globbed by a job, then deselected by every one of their -k expressions",
)
for path in globbed
if not allowlist.covers_test(path)
and not any(_token_covers(token, path) for token in named_by_workflow)
and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
)
def _holds_tests(directory: pathlib.Path) -> bool:
return any(directory.rglob("test_*.py"))
def _shard_children(root: str, repo_root: pathlib.Path = REPO_ROOT) -> tuple[str, ...]:
"""Children of a sharded root that carry tests, so each one needs its own shard.
A directory earns an entry by containing a test file rather than by being named
`test_*`, which is what keeps fixture directories (`test_configs`, `expected_*`)
out without a hand-maintained list of exceptions.
"""
return tuple(
sorted(
child.relative_to(repo_root).as_posix()
for child in (repo_root / root).iterdir()
if not child.name.startswith(".")
and (
_holds_tests(child)
if child.is_dir()
else child.name.startswith("test_") and child.suffix == ".py"
)
)
)
def _unassigned_shard_children(
tokens: frozenset[str],
roots: tuple[str, ...] = SHARDED_ROOTS,
repo_root: pathlib.Path = REPO_ROOT,
) -> tuple[Finding, ...]:
return tuple(
Finding(subject=child, detail=f"holds tests but no shard of {root} names it")
for root in roots
if (repo_root / root).is_dir()
for child in _shard_children(root, repo_root)
if child not in roots and not any(_token_names(token, child) for token in tokens)
)
def _uncovered_dockerfiles(allowlist: Allowlist, tokens: frozenset[str]) -> tuple[Finding, ...]:
return tuple(
Finding(subject=relative_path, detail="built by no job")
@ -174,6 +393,25 @@ def _uncovered_dockerfiles(allowlist: Allowlist, tokens: frozenset[str]) -> tupl
)
def _stale_allowlist_paths(
allowlist: Allowlist,
*,
test_files: tuple[str, ...],
dockerfiles: tuple[str, ...],
) -> tuple[Finding, ...]:
sections: Final[tuple[Section, ...]] = (
Section("test_paths", allowlist.test_paths, test_files, _token_covers),
Section("dockerfiles", allowlist.dockerfiles, dockerfiles, operator.eq),
)
return tuple(
Finding(subject=path, detail=f"listed under '{section.name}' but matches no file the census looks at")
for section in sections
for entry in section.entries
for path in entry.paths
if not any(section.matches(path, candidate) for candidate in section.candidates)
)
def _parse_entry(item: object, section: str) -> AllowEntry:
if not isinstance(item, dict):
raise SystemExit(f"{ALLOWLIST_FILE.name}: '{section}' entries must be mappings")
@ -229,13 +467,56 @@ def _report(title: str, findings: tuple[Finding, ...], remedy: str) -> None:
_write("")
def _check_slices() -> int:
findings: Final = _deselected_everywhere(_load_allowlist())
if findings:
_report(
"test files a -k expression removes from every job that globs them",
findings,
"Give each one a job whose -k keeps it, or list it in "
".github/ci-coverage-allowlist.yml with the reason it may stay unrun.",
)
return 1
_write(f"OK: no test file is globbed by a job and then deselected by every -k across {len(_slices())} slices.")
return 0
def _check_shards() -> int:
findings = _unassigned_shard_children(_invoked_test_tokens(_all_scalars()))
if findings:
_report(
"test directories and files that no shard claims",
findings,
"Add each to the shard it belongs to. A directory that is itself split across "
"several shards belongs in SHARDED_ROOTS instead, so its own children get checked.",
)
return 1
counted = sum(len(_shard_children(root)) for root in SHARDED_ROOTS if (REPO_ROOT / root).is_dir())
_write(f"OK: all {counted} test children across {len(SHARDED_ROOTS)} sharded trees are assigned to a shard.")
return 0
def main() -> int:
if "--shards" in sys.argv[1:]:
return _check_shards()
if "--slices" in sys.argv[1:]:
return _check_slices()
allowlist = _load_allowlist()
scalars = _all_scalars()
test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars))
dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
if stale_findings:
_report(
"allowlist entries that exempt nothing",
stale_findings,
"Delete each from .github/ci-coverage-allowlist.yml; the file it named is gone or was renamed.",
)
if test_findings:
_report(
"test files that no CI job invokes",
@ -248,7 +529,7 @@ def main() -> int:
dockerfile_findings,
"Build each in a workflow, or list it in .github/ci-coverage-allowlist.yml with a reason.",
)
if test_findings or dockerfile_findings:
if stale_findings or test_findings or dockerfile_findings:
return 1
_write(

149
.github/scripts/assert_workflow_dir_hygiene.py vendored Normal file
View file

@ -0,0 +1,149 @@
#!/usr/bin/env python3
"""Three invariants about what lives in .github/workflows/ and what its names mean.
`.github/workflows/` is a directory GitHub reads, not a place to keep things. Every
file at its top level is parsed as a workflow, so a script or a data file parked there
is either an invalid workflow or an orphan nobody can find. A subdirectory is not read
at all, so helper files may live in one. GitHub accepts both `.yml` and `.yaml`, and
this repo spells them `.yml`, which is a naming rule rather than a validity one and is
reported separately. And the `_` prefix is the repo's only signal that a workflow is a
reusable building block rather than something that runs on its own, which is worth
nothing unless it is true both ways.
WF001 a top-level file in .github/workflows/ that is not a workflow at all
WF002 a workflow whose only trigger is `workflow_call` but is not `_`-prefixed
WF003 a `_`-prefixed workflow that no other workflow can call
WF004 a real workflow spelled `.yaml` where this directory spells them `.yml`
A workflow with `workflow_call` alongside a human trigger is deliberately dual-mode
and belongs under its plain name, so only the call-only ones are held to WF002.
Usage
-----
python assert_workflow_dir_hygiene.py
Exit code 1 if any violation is found.
"""
from __future__ import annotations
import pathlib
import sys
from dataclasses import dataclass
from typing import Final
import yaml
REPO_ROOT: Final = pathlib.Path(__file__).resolve().parents[2]
WORKFLOW_DIR: Final = REPO_ROOT / ".github" / "workflows"
SCRIPT_HOME: Final = ".github/scripts/"
REUSABLE_PREFIX: Final = "_"
CALL_TRIGGER: Final = "workflow_call"
CANONICAL_SUFFIX: Final = ".yml"
WORKFLOW_SUFFIXES: Final = frozenset((CANONICAL_SUFFIX, ".yaml"))
@dataclass(frozen=True, slots=True)
class Finding:
subject: str
code: str
detail: str
def render(self) -> str:
return f" - {self.subject}: {self.code} {self.detail}"
def _triggers(document: object) -> frozenset[str]:
if not isinstance(document, dict):
return frozenset()
raw: Final = document.get("on", document.get(True))
if isinstance(raw, str):
return frozenset({raw})
if isinstance(raw, dict):
return frozenset(str(key) for key in raw)
if isinstance(raw, list):
return frozenset(str(item) for item in raw)
return frozenset()
def _workflows(directory: pathlib.Path) -> tuple[pathlib.Path, ...]:
return tuple(
path
for path in sorted(directory.iterdir())
if path.is_file() and path.suffix in WORKFLOW_SUFFIXES
)
def _strays(directory: pathlib.Path) -> tuple[Finding, ...]:
return tuple(
Finding(
path.name,
"WF001",
f"is not a workflow, and GitHub parses every top-level file here as one; "
f"move it to {SCRIPT_HOME} or into a subdirectory, which GitHub does not read",
)
for path in sorted(directory.iterdir())
if path.is_file() and path.suffix not in WORKFLOW_SUFFIXES
)
def _misspelled(directory: pathlib.Path) -> tuple[Finding, ...]:
return tuple(
Finding(
path.name,
"WF004",
f"is a real workflow and GitHub reads it, but this directory spells them "
f"{CANONICAL_SUFFIX}; rename it to {path.stem}{CANONICAL_SUFFIX}",
)
for path in _workflows(directory)
if path.suffix != CANONICAL_SUFFIX
)
def _misnamed(directory: pathlib.Path) -> tuple[Finding, ...]:
return tuple(
finding
for path in _workflows(directory)
for finding in _naming_findings(path, _triggers(yaml.safe_load(path.read_text(encoding="utf-8"))))
)
def _naming_findings(path: pathlib.Path, triggers: frozenset[str]) -> tuple[Finding, ...]:
underscored: Final = path.name.startswith(REUSABLE_PREFIX)
if triggers == frozenset({CALL_TRIGGER}) and not underscored:
return (
Finding(
path.name,
"WF002",
f"is only callable by another workflow, so name it {REUSABLE_PREFIX}{path.name}",
),
)
if underscored and CALL_TRIGGER not in triggers:
return (
Finding(
path.name,
"WF003",
f"is named as a reusable workflow but has no {CALL_TRIGGER} trigger; "
"add one or drop the prefix",
),
)
return ()
def main() -> int:
findings: Final = _strays(WORKFLOW_DIR) + _misspelled(WORKFLOW_DIR) + _misnamed(WORKFLOW_DIR)
if not findings:
total: Final = len(_workflows(WORKFLOW_DIR))
sys.stdout.write(
f"OK: {total} workflows, every file in .github/workflows/ is one, and the "
f"{REUSABLE_PREFIX} prefix means callable in both directions.\n"
)
return 0
sys.stdout.write("ERROR: .github/workflows/ holds files that break its own conventions\n")
for finding in findings:
sys.stdout.write(f"{finding.render()}\n")
return 1
if __name__ == "__main__":
sys.exit(main())

0
.github/workflows/auto_update_price_and_context_window_file.py → .github/scripts/auto_update_price_and_context_window_file.py vendored
View file

42
.github/scripts/detect_changes.sh vendored Executable file
View file

@ -0,0 +1,42 @@
#!/usr/bin/env bash
set -uo pipefail
readonly API_FILE_CEILING=3000
readonly CATEGORY="${CATEGORY:-backend}"
decide() {
echo "detect-changes[${CATEGORY}]: decision=$1"
[ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=$1" >>"${GITHUB_OUTPUT}"
exit 0
}
run_full() {
echo "detect-changes[${CATEGORY}]: $1; running job"
decide run
}
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
classify="${here}/../../.circleci/scripts/classify_changes.sh"
[ -n "${PR_NUMBER:-}" ] || run_full "not a pull_request event"
[ -n "${REPO:-}" ] || run_full "no repository in the environment"
case "${CHANGED_FILE_COUNT:-}" in
'' | *[!0-9]*) run_full "the event payload carries no changed_files count" ;;
esac
[ "${CHANGED_FILE_COUNT}" -le "${API_FILE_CEILING}" ] ||
run_full "PR #${PR_NUMBER} changes ${CHANGED_FILE_COUNT} files, past the ${API_FILE_CEILING}-file listing ceiling"
changed="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename')" ||
run_full "could not list the files on PR #${PR_NUMBER}"
[ -n "${changed}" ] || run_full "the API listed no files on PR #${PR_NUMBER}"
echo "detect-changes[${CATEGORY}]: files changed by PR #${PR_NUMBER}:"
printf '%s\n' "${changed}" | sed 's/^/ /'
decision="$(printf '%s\n' "${changed}" | bash "${classify}" "${CATEGORY}")" ||
run_full "classify_changes.sh failed"
case "${decision}" in
run | skip) decide "${decision}" ;;
*) run_full "classify_changes.sh printed an unexpected decision: ${decision}" ;;
esac

198
.github/scripts/e2e_egress_sentinel.py vendored Executable file
View file

@ -0,0 +1,198 @@
"""Prove an e2e replay run makes zero outbound provider calls, by counting them.
`serve` pins each provider host (`--host`) to a local sink address in the hosts
file and binds a counting listener on that address, so any connection the proxy
or the record/replay edge opens to a real provider is redirected to the sink,
recorded as one line in `--hits-file`, and never leaves the box. The record and
replay edge only ever dials `127.0.0.1:<edge-port>` (a different host than the
pinned provider names), so in a clean replay the sink sees nothing; a single hit
means a provider call escaped the bundle. `assert-empty` turns that hit file into
the pass/fail check.
Stdlib only, so CI runs it under the system interpreter as root (binding :443 and
editing the hosts file both need root); `--sink-address`, `--port`, and
`--hosts-file` are injectable so it runs unprivileged against a temp hosts file on
a high port under test.
"""
# ruff: noqa: T201 # CLI script: its stdout/stderr progress and results are the interface
from __future__ import annotations
import argparse
import json
import os
import signal
import socket
import sys
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from types import FrameType
from typing import Final
_BLOCK_BEGIN: Final = "# BEGIN e2e-egress-sentinel"
_BLOCK_END: Final = "# END e2e-egress-sentinel"
@dataclass(frozen=True, slots=True)
class ServeConfig:
hosts: tuple[str, ...]
sink_address: str
ports: tuple[int, ...]
hits_file: Path
hosts_file: Path
ready_file: Path | None
pid_file: Path | None
def _pin_block(sink_address: str, hosts: tuple[str, ...]) -> str:
lines = "\n".join(f"{sink_address}\t{host}" for host in hosts)
return f"\n{_BLOCK_BEGIN}\n{lines}\n{_BLOCK_END}\n"
def _install_pins(hosts_file: Path, sink_address: str, hosts: tuple[str, ...]) -> bytes:
original = hosts_file.read_bytes() if hosts_file.exists() else b""
hosts_file.write_bytes(original + _pin_block(sink_address, hosts).encode())
return original
def _restore_pins(hosts_file: Path, original: bytes) -> None:
hosts_file.write_bytes(original)
def _bind(sink_address: str, port: int) -> socket.socket:
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind((sink_address, port))
listener.listen(128)
return listener
@dataclass(frozen=True, slots=True)
class _HitLog:
path: Path
_lock: threading.Lock
def record(self, *, port: int, peer: tuple[str, int]) -> None:
entry = json.dumps({"ts": time.time(), "port": port, "peer": list(peer)})
with self._lock:
with self.path.open("a", encoding="utf-8") as handle:
handle.write(entry + "\n")
def _serve_socket(listener: socket.socket, port: int, hits: _HitLog, stop: threading.Event) -> None:
while not stop.is_set():
try:
conn, peer = listener.accept()
except OSError:
return
hits.record(port=port, peer=(peer[0], peer[1]))
try:
conn.close()
except OSError:
pass
def serve(config: ServeConfig) -> int:
config.hits_file.write_text("", encoding="utf-8")
original_hosts = _install_pins(config.hosts_file, config.sink_address, config.hosts)
try:
listeners = tuple(_bind(config.sink_address, port) for port in config.ports)
except OSError as exc:
_restore_pins(config.hosts_file, original_hosts)
print(f"egress sentinel could not bind a sink: {exc}", file=sys.stderr)
return 1
stop = threading.Event()
hits = _HitLog(path=config.hits_file, _lock=threading.Lock())
threads = tuple(
threading.Thread(target=_serve_socket, args=(listener, port, hits, stop), daemon=True)
for listener, port in zip(listeners, config.ports)
)
for thread in threads:
thread.start()
def _handle(_signum: int, _frame: FrameType | None) -> None:
stop.set()
for listener in listeners:
try:
listener.close()
except OSError:
pass
signal.signal(signal.SIGTERM, _handle)
signal.signal(signal.SIGINT, _handle)
if config.pid_file is not None:
config.pid_file.write_text(str(os.getpid()), encoding="utf-8")
if config.ready_file is not None:
config.ready_file.write_text("ready", encoding="utf-8")
print(
f"egress sentinel up: pinned {', '.join(config.hosts)} to {config.sink_address} "
f"on port(s) {', '.join(str(p) for p in config.ports)}",
flush=True,
)
stop.wait()
_restore_pins(config.hosts_file, original_hosts)
if config.ready_file is not None and config.ready_file.exists():
config.ready_file.unlink()
if config.pid_file is not None and config.pid_file.exists():
config.pid_file.unlink()
return 0
def assert_empty(hits_file: Path) -> int:
if not hits_file.exists():
print(f"egress sentinel recorded no provider calls ({hits_file} absent): zero egress")
return 0
hits = [line for line in hits_file.read_text(encoding="utf-8").splitlines() if line.strip()]
if not hits:
print("egress sentinel recorded no provider calls: zero egress")
return 0
print(f"egress sentinel recorded {len(hits)} provider call(s); replay was not hermetic:", file=sys.stderr)
for line in hits:
print(f" {line}", file=sys.stderr)
return 1
def _serve_from_args(args: argparse.Namespace) -> int:
config = ServeConfig(
hosts=tuple(args.host),
sink_address=args.sink_address,
ports=tuple(args.port),
hits_file=Path(args.hits_file),
hosts_file=Path(args.hosts_file),
ready_file=Path(args.ready_file) if args.ready_file else None,
pid_file=Path(args.pid_file) if args.pid_file else None,
)
return serve(config)
def main(argv: tuple[str, ...]) -> int:
parser = argparse.ArgumentParser(description="count outbound provider calls during an e2e replay")
sub = parser.add_subparsers(dest="command", required=True)
serve_parser = sub.add_parser("serve", help="pin provider hosts and count connection attempts")
serve_parser.add_argument("--host", action="append", required=True, help="provider host to pin and watch")
serve_parser.add_argument("--sink-address", default="127.0.0.1")
serve_parser.add_argument("--port", action="append", type=int, default=None)
serve_parser.add_argument("--hits-file", required=True)
serve_parser.add_argument("--hosts-file", default="/etc/hosts")
serve_parser.add_argument("--ready-file", default=None)
serve_parser.add_argument("--pid-file", default=None)
assert_parser = sub.add_parser("assert-empty", help="exit non-zero if any provider call was recorded")
assert_parser.add_argument("--hits-file", required=True)
args = parser.parse_args(argv)
if args.command == "serve":
if args.port is None:
args.port = [443]
return _serve_from_args(args)
return assert_empty(Path(args.hits_file))
if __name__ == "__main__":
raise SystemExit(main(tuple(sys.argv[1:])))

55
.github/scripts/e2e_fetch_fixture_bundle.sh vendored Executable file
View file

@ -0,0 +1,55 @@
#!/usr/bin/env bash
set -euo pipefail
REPO="${1:-${GITHUB_REPOSITORY:?REPO required}}"
ARTIFACT_NAME="${2:-e2e-fixtures-bundle}"
BASE_BRANCH="${3:?base branch required}"
DEST_DIR="${4:?destination bundle dir required}"
: "${GH_TOKEN:?GH_TOKEN required to query and download artifacts}"
WORKDIR="$(mktemp -d)"
trap 'rm -rf "${WORKDIR}"' EXIT
echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
SELECTED="$(
gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
--jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
| jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
)"
if [[ -z "${SELECTED}" ]]; then
echo "no usable '${ARTIFACT_NAME}' artifact on ${BASE_BRANCH}: the last record run produced none (a red Saturday), so there is nothing fresh to replay; failing loudly instead of replaying a stale bundle" >&2
exit 1
fi
RUN_ID="$(echo "${SELECTED}" | jq -r '.run_id')"
RUN_NUMBER="$(echo "${SELECTED}" | jq -r '.run_number')"
ARTIFACT_ID="$(echo "${SELECTED}" | jq -r '.id')"
GH_DIGEST="$(echo "${SELECTED}" | jq -r '.digest // "unknown"')"
CREATED_AT="$(echo "${SELECTED}" | jq -r '.created_at')"
echo "pinned bundle: run #${RUN_NUMBER} (run_id=${RUN_ID}, artifact_id=${ARTIFACT_ID}), recorded ${CREATED_AT}, github digest ${GH_DIGEST}"
gh run download "${RUN_ID}" --repo "${REPO}" -n "${ARTIFACT_NAME}" -D "${WORKDIR}"
TARBALL="$(find "${WORKDIR}" -name '*.tar.gz' -type f | head -n 1)"
if [[ -z "${TARBALL}" ]]; then
echo "downloaded artifact contained no tarball" >&2
exit 1
fi
SIDECAR="${TARBALL}.sha256"
if [[ ! -f "${SIDECAR}" ]]; then
echo "downloaded artifact has no ${SIDECAR}: cannot verify the bundle digest" >&2
exit 1
fi
echo "verifying bundle against its recorded sha256 digest"
( cd "$(dirname "${TARBALL}")" && sha256sum -c "$(basename "${SIDECAR}")" )
mkdir -p "${DEST_DIR}"
tar xzf "${TARBALL}" -C "${DEST_DIR}"
echo "extracted bundle into ${DEST_DIR}"
python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print(' recorded_at', m['recorded_at'], 'harness', m['harness_version'], 'format_version', m['format_version'])" "${DEST_DIR}/manifest.json"

36
.github/scripts/e2e_pack_fixture_bundle.sh vendored Executable file
View file

@ -0,0 +1,36 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -ne 2 ]]; then
echo "usage: $0 <bundle-dir> <out-tarball>" >&2
exit 2
fi
BUNDLE_DIR="$1"
OUT_TARBALL="$2"
MANIFEST="${BUNDLE_DIR}/manifest.json"
if [[ ! -f "${MANIFEST}" ]]; then
echo "no ${MANIFEST}: refusing to publish a bundle with no manifest (record produced nothing)" >&2
exit 1
fi
echo "packing fixture bundle from ${BUNDLE_DIR}"
python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print(' format_version', m['format_version'], 'recorded_at', m['recorded_at'], 'harness', m['harness_version'])" "${MANIFEST}"
TEST_DIRS=$(find "${BUNDLE_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
if [[ "${TEST_DIRS}" -eq 0 ]]; then
echo "bundle at ${BUNDLE_DIR} has a manifest but no recorded interactions; refusing to publish an empty bundle" >&2
exit 1
fi
echo " ${TEST_DIRS} recorded test director(ies)"
mkdir -p "$(dirname "${OUT_TARBALL}")"
tar czf "${OUT_TARBALL}" -C "${BUNDLE_DIR}" .
OUT_DIR="$(cd "$(dirname "${OUT_TARBALL}")" && pwd)"
OUT_BASE="$(basename "${OUT_TARBALL}")"
( cd "${OUT_DIR}" && sha256sum "${OUT_BASE}" > "${OUT_BASE}.sha256" )
echo "wrote ${OUT_TARBALL} ($(du -h "${OUT_TARBALL}" | cut -f1)) and ${OUT_BASE}.sha256"
cat "${OUT_DIR}/${OUT_BASE}.sha256"

0
.github/workflows/run_llm_translation_tests.py → .github/scripts/run_llm_translation_tests.py vendored
View file

15
.github/scripts/select_ui_test_scope.sh vendored Executable file
View file

@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -uo pipefail
has_file=false
has_file_outside_src=false
while IFS= read -r file || [ -n "$file" ]; do
[ -n "$file" ] || continue
has_file=true
case "$file" in
src/*) ;;
*) has_file_outside_src=true ;;
esac
done
{ [ "$has_file" = true ] && [ "$has_file_outside_src" = false ]; } && echo related || echo full

557
.github/scripts/triage_rollout_heads_up.py vendored
View file

@ -1,557 +0,0 @@
#!/usr/bin/env python3
"""One-shot 7-day heads-up sweep for the Agent Shin rollout.
Posts a friendly "the OSS triage bot kicks in next Monday" comment on every
open external PR/issue that currently *would* fail the new rubric — i.e.,
every PR/issue Agent Shin would close once the rollout completes. The point
is to give contributors a full week to fix their description before the bot
ever takes a destructive action, so nobody is surprised by an auto-close.
The script is designed to run **exactly once** at rollout, fired by a manual
``workflow_dispatch`` (``dry_run=false``) on the heads-up workflow. Re-runs
are safe: every comment is stamped with the hidden ``HEADS_UP_MARKER`` and
PRs/issues that already carry the marker are skipped.
Dry-run vs. real run
--------------------
Defaults to dry-run. Passing ``--close`` flips into real mode. Every GitHub
mutation goes through ``_agent_shin_actions``, which has a one-line
``if dry_run: log else: do_it`` per call, so the only difference between a
dry-run preview and the real run is the call site that actually hits the
GitHub API.
Local preview::
python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm
Real run (the manual rollout dispatch uses this)::
python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm --close
"""
from __future__ import annotations
import argparse
import datetime as dt
import json
import os
import sys
from pathlib import Path
from typing import Any
# Make the sibling triage_with_llm + _agent_shin_actions importable when this
# script is invoked directly (the GitHub workflow does `python3 .github/scripts/...`).
_SCRIPTS_DIR = Path(__file__).resolve().parent
if str(_SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(_SCRIPTS_DIR))
from _agent_shin_actions import maybe_post_comment # noqa: E402
from agent_shin_shared import ( # noqa: E402
AGENT_SHIN_DEFAULT_BOT_LOGIN,
ALLOWLIST_LOGINS,
list_open_items,
)
from triage_with_llm import ( # noqa: E402
DEFAULT_MODEL,
call_llm_judge,
fetch_issue,
fetch_pr,
gh,
is_internal_contributor,
review_gate,
triage,
)
# Hidden marker so re-runs skip PRs/issues we've already notified. Distinct from
# the within-grace / ready / regressed markers so it can't be confused with the
# steady-state lifecycle comments.
HEADS_UP_MARKER = "<!-- agent-shin:rollout-heads-up -->"
# Placeholder until the litellm-docs PR ships. The rollout blog post explains
# the new rubric, the 7-day grace, and how to recover after an auto-close.
# TODO(docs): replace with the canonical URL once the litellm-docs PR merges.
ROLLOUT_BLOG_URL = "https://docs.litellm.ai/docs/agent_shin_triage_rollout"
# Default cutoff is one week from "now". Computed at runtime so the wording
# stays correct even if the rollout is merged later than planned. The user can
# override with --close-on YYYY-MM-DD when running the script manually.
DEFAULT_GRACE_DAYS = 7
# The daily auto-close sweeps (close_low_quality_prs.yml at 09:00 UTC and
# review_gate.yml at 09:30 UTC) are what actually close a still-failing item,
# so the deadline we promise contributors has to name that wall-clock moment.
ACTIVATION_TIME_UTC = "09:00 UTC"
def _format_cutoff(cutoff: dt.date) -> str:
"""Human-readable, timezone-explicit cutoff, e.g. ``Monday, June 1, 2026
(09:00 UTC)`` — the moment a still-failing PR/issue gets closed."""
return (
f"{cutoff.strftime('%A, %B')} {cutoff.day}, {cutoff.year} "
f"({ACTIVATION_TIME_UTC})"
)
def _rubric_section_pr() -> str:
return (
"**Going forward, every external PR needs ONE of:**\n"
"\n"
"- A linked GitHub issue using a closing keyword: "
"`Fixes #1234`, `Closes #1234`, or `Resolves #1234`, OR\n"
"- All three of: a clear **problem description**, **expected vs. "
"actual behavior**, and **end-to-end QA proof** (at least one of a "
"short screen recording / video, before/after screenshots, or the "
"exact commands you ran with their real output; mocked or stubbed "
"runs don't count).\n"
"\n"
"PRs also need a **Greptile confidence score of 4/5 or higher** before "
"the bot will tag them `ready for review`. You can `@greptileai` to "
"request a fresh review at any time, including after the PR is closed."
)
def _rubric_section_issue() -> str:
return (
"**Going forward, every external issue needs:**\n"
"\n"
"- For **bug reports**: end-to-end evidence of the bug (at least one "
"of a screen recording / video, a screenshot, or the exact commands "
"you ran with their real output / traceback) plus expected vs. actual "
"behavior. Written steps with no run output don't count, and mocked "
"or stubbed runs don't count.\n"
"- For **feature requests**: a clear description of the proposed "
"feature plus a use case + concrete example (config, API call, UI "
"flow, or scenario showing what's blocked today)."
)
def _description_only_note(kind: str) -> str:
noun = "PR" if kind == "pr" else "issue"
return (
f"⚠️ **The requirements must live in the {noun} *description*, not in "
"comments.** Some PRs/issues collect 100+ comments from humans and "
"bots; reading the entire thread on every triage run would balloon "
"GitHub API usage (we'd start getting 429'd) and blow out the LLM "
"judge's context. The bot only reads the description, so anything "
"you add as a comment will be invisible to it."
)
def _missing_section(verdict: dict, greptile_score: int | None) -> str:
"""Bullet list of what's currently missing on this PR/issue.
Combines the LLM judge's `missing` list (rubric items) with a Greptile
shortfall (for PRs) so the contributor sees one list of things to fix.
"""
missing = list(verdict.get("missing") or [])
if greptile_score is not None and greptile_score < 4:
missing.insert(
0,
f"Greptile's most recent review scored this PR {greptile_score}/5 "
"(below the 4/5 bar Agent Shin will require).",
)
if not missing:
return (
"_The bot couldn't articulate a specific missing piece; see the "
"rubric link above and double-check the description includes all "
"of it before the rollout._"
)
bullets = "\n".join(f"- {m}" for m in missing)
return f"**What this one is currently missing:**\n\n{bullets}"
def _recovery_section(kind: str) -> str:
if kind == "pr":
return (
"**If the bot closes this PR after the rollout:** update the "
"description with the missing pieces, then either open a fresh "
"PR or comment `@agent-shin reconsider` on the closed PR. If "
"Greptile re-scores you at 4/5 or higher I'll reopen and tag "
"the PR `ready for review`. (`@greptileai` works on closed PRs "
"too; a fresh review is one of the signals that lifts you back "
"into the queue.) This is **not** us losing interest in your "
"change; far from it. We just need open PRs to be a list of "
"things a maintainer can act on, so we can get to yours faster."
)
return (
"**If the bot closes this issue after the rollout:** edit the issue "
"description to add the missing pieces, then comment `@agent-shin "
"reconsider` on the closed issue. I'll re-evaluate and, if the rubric "
"is met, reopen it. (GitHub doesn't let external authors reopen an "
"issue a maintainer or bot closed, so the comment is the reliable "
"path.) This is **not** us saying the bug isn't real or the request "
"isn't useful; it's so the remaining open issues are a list of things "
"a maintainer can act on."
)
def format_heads_up_comment(
*, kind: str, verdict: dict, greptile_score: int | None, cutoff: dt.date
) -> str:
"""Compose the friendly 7-day heads-up comment posted on a failing PR/issue."""
noun = "PR" if kind == "pr" else "issue"
rubric = _rubric_section_pr() if kind == "pr" else _rubric_section_issue()
cutoff_str = _format_cutoff(cutoff)
explanation = (verdict.get("explanation") or "").strip()
explanation_block = (
f"> _(The judge's note for this one: {explanation})_\n\n" if explanation else ""
)
return (
"🚅 **Heads-up: we're turning on the OSS triage bot in "
f"{DEFAULT_GRACE_DAYS} days, on {cutoff_str}.**\n"
"\n"
"We're rolling out **Agent Shin**, an LLM-as-judge triage bot for "
f"external {noun}s. Once it's live, the bot reads each open "
f"{noun}'s description, scores it against a small rubric, and "
f"auto-closes any {noun} that's missing the basics, with a single "
f"comment explaining what's missing and how to recover. Full "
f"context: [Agent Shin rollout blog post]({ROLLOUT_BLOG_URL}).\n"
"\n"
f"{rubric}\n"
"\n"
f"{_description_only_note(kind)}\n"
"\n"
f"{_missing_section(verdict, greptile_score)}\n"
"\n"
f"{explanation_block}"
"**Timeline (you have a week):**\n"
"\n"
f"- We turn the bot on in {DEFAULT_GRACE_DAYS} days, on "
f"**{cutoff_str}**. You have until then to update this {noun}'s "
"description with the missing pieces above.\n"
f"- If this {noun} still fails the rubric at **{cutoff_str}**, "
"we'll close it.\n"
f"- From then on the bot runs daily, and every {noun} that fails "
"the rubric gets a **2-hour lifetime**: one warning comment, then "
"auto-close 2 hours later.\n"
"\n"
f"{_recovery_section(kind)}\n"
"\n"
f"{HEADS_UP_MARKER}"
)
def _list_open_numbers(repo: str, kind: str) -> list[int]:
"""Return every open PR or issue number in ``repo``.
Delegates to ``list_open_items`` so the full backlog is fetched (no cap)
and the `gh {pr,issue} list` invocation stays in one shared place. ``gh
issue list`` would include PRs, but ``list_open_items`` uses the dedicated
command per kind, so the two never mix.
"""
return [
item["number"] for item in list_open_items(kind, repo=repo, fields="number")
]
def _has_heads_up_marker(item: dict) -> bool:
"""Cheap fast-path: check the PR/issue body itself for the marker.
The marker is appended to the *comment* we post, not the body, so this
will only fire if the body literally contains the marker text. We still
do the comment-marker check separately below; this body check just lets
us short-circuit for PRs/issues that quote the marker for any reason.
"""
body = item.get("body") or ""
return HEADS_UP_MARKER in body
def _comments_have_marker(repo: str, number: int) -> bool:
"""True if the bot already posted a comment carrying the marker.
Used for idempotency: a re-run skips items the previous run notified.
Filters by author (matching the sibling marker-checks in
``triage_with_llm._has_marker`` and
``agent_shin_shared.seconds_since_latest_marker_comment``) so a
contributor who quotes the heads-up via GitHub's "Quote reply" — which
preserves HTML comments in the raw markdown — can't trick the
idempotency check into silently skipping a real heads-up.
Comments live on the unified issues endpoint regardless of whether the
item is a PR or an issue, so no ``kind`` argument is required here.
"""
expected_login = (
os.environ.get("AGENT_SHIN_BOT_LOGIN") or AGENT_SHIN_DEFAULT_BOT_LOGIN
).lower()
raw = gh(
"api",
"--paginate",
f"repos/{repo}/issues/{number}/comments?per_page=100",
)
for line in raw.splitlines():
line = line.strip()
if not line:
continue
try:
payload = json.loads(line)
except json.JSONDecodeError:
continue
comments = payload if isinstance(payload, list) else [payload]
for comment in comments:
author = ((comment.get("user") or {}).get("login") or "").lower()
if author != expected_login:
continue
if HEADS_UP_MARKER in (comment.get("body") or ""):
return True
return False
def _evaluate_pr(*, repo: str, number: int, model: str, judge: Any = None) -> dict:
"""Run the future PR rubric (review_gate) in dry-run and return the result."""
return review_gate(
repo=repo,
number=number,
close=False, # we only want the verdict, never act here
model=model,
judge=judge,
)
def _evaluate_issue(*, repo: str, number: int, model: str, judge: Any = None) -> dict:
"""Run the future issue rubric (triage kind='issue') in dry-run."""
return triage(
repo=repo,
kind="issue",
number=number,
close=False,
model=model,
judge=judge,
)
def _would_be_closed(kind: str, result: dict) -> bool:
"""True if the future triage would auto-close this PR/issue based on the
rubric (regardless of grace-period gating).
For PRs we trust ``review_gate``'s ``passing`` field — it combines the LLM
verdict and the Greptile score. For issues we read the LLM verdict
directly. Both fields are ``None``/missing on skip paths
(skip-internal-author, skip-llm-error, etc.) where the future bot would
NOT close the item — those return False.
"""
if kind == "pr":
passing = result.get("passing")
if passing is None:
return False # skipped — nothing for the heads-up to warn about
return passing is False
verdict = result.get("verdict") or {}
return (verdict.get("verdict") or "").lower() == "fail"
def _process_one(
*,
repo: str,
kind: str,
number: int,
model: str,
cutoff: dt.date,
dry_run: bool,
judge: Any = None,
skip_marker_check: bool = False,
allowlist: frozenset[str] = ALLOWLIST_LOGINS,
) -> dict:
"""Evaluate one PR/issue and post a heads-up if it would be auto-closed.
Returns a per-item dict for the summary table.
"""
base = {"kind": kind, "number": number}
fetcher = fetch_pr if kind == "pr" else fetch_issue
item = fetcher(repo, number)
if (item.get("state") or "") != "open":
return {**base, "action": "skip-not-open"}
if allowlist:
login = (item.get("user") or {}).get("login") or ""
if login.lower() not in allowlist:
return {**base, "action": "skip-not-allowlisted"}
elif is_internal_contributor(item):
return {**base, "action": "skip-internal-author"}
if not skip_marker_check and _has_heads_up_marker(item):
return {**base, "action": "skip-already-marked-in-body"}
if not skip_marker_check and _comments_have_marker(repo, number):
return {**base, "action": "skip-already-notified"}
if kind == "pr":
result = _evaluate_pr(repo=repo, number=number, model=model, judge=judge)
else:
result = _evaluate_issue(repo=repo, number=number, model=model, judge=judge)
if not _would_be_closed(kind, result):
return {**base, "action": "skip-passing", "evaluator": result.get("action")}
verdict = result.get("verdict") or {}
greptile_score = result.get("greptile_score") if kind == "pr" else None
comment = format_heads_up_comment(
kind=kind, verdict=verdict, greptile_score=greptile_score, cutoff=cutoff
)
maybe_post_comment(repo, number, comment, dry_run=dry_run)
return {
**base,
"action": "heads-up-posted" if not dry_run else "would-post-heads-up",
"verdict": (verdict.get("verdict") or "").lower(),
"greptile_score": greptile_score,
}
def _print_summary(results: list[dict]) -> None:
"""Tally per-action counts so a dry-run preview tells you at a glance how
many comments the real run would post."""
counts: dict[str, int] = {}
for r in results:
counts[r["action"]] = counts.get(r["action"], 0) + 1
print("\n=== rollout heads-up summary ===")
for action in sorted(counts):
print(f" {action:35s} {counts[action]}")
print(f" total {len(results)}")
def run(
*,
repo: str,
close: bool,
cutoff: dt.date,
model: str,
kinds: tuple[str, ...] = ("pr", "issue"),
judge: Any = None,
only_numbers: dict[str, list[int]] | None = None,
skip_marker_check: bool = False,
) -> list[dict]:
"""Sweep ``repo`` and post heads-up comments. Returns the per-item results."""
dry_run = not close
if dry_run:
print(
f"[DRY RUN] sweeping {repo}; --close not passed, no comments will be posted."
)
else:
print(f"[REAL RUN] sweeping {repo}; comments WILL be posted.")
print(f"Cutoff date in comment body: {cutoff.isoformat()}")
results: list[dict] = []
for kind in kinds:
if only_numbers and kind in only_numbers:
numbers = list(only_numbers[kind])
else:
numbers = _list_open_numbers(repo, kind)
print(f"\n--- {kind}s: {len(numbers)} open ---")
for n in numbers:
try:
result = _process_one(
repo=repo,
kind=kind,
number=n,
model=model,
cutoff=cutoff,
dry_run=dry_run,
judge=judge,
skip_marker_check=skip_marker_check,
)
except (
Exception
) as exc: # noqa: BLE001 - per-item errors don't abort the sweep
result = {
"kind": kind,
"number": n,
"action": "error",
"error": str(exc),
}
print(f"!! {kind}#{n}: {exc}", file=sys.stderr)
print(f" {kind}#{n}: {result['action']}")
results.append(result)
_print_summary(results)
return results
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo", required=True, help="owner/repo")
parser.add_argument(
"--close",
action="store_true",
help=(
"Actually post comments. Without this flag the script is in "
"dry-run mode and only logs what it would do."
),
)
parser.add_argument(
"--close-on",
type=dt.date.fromisoformat,
default=None,
help=(
"Cutoff date shown in the heads-up comment as the rollout date "
f"(default: today + {DEFAULT_GRACE_DAYS} days)."
),
)
parser.add_argument(
"--model",
default=os.environ.get("TRIAGE_MODEL") or DEFAULT_MODEL,
help=f"Model for the rubric LLM judge (default: {DEFAULT_MODEL}).",
)
parser.add_argument(
"--kind",
choices=("pr", "issue", "both"),
default="both",
help="Restrict the sweep to PRs or issues only (default: both).",
)
parser.add_argument(
"--only-pr",
type=int,
action="append",
default=[],
help="Limit the PR sweep to these PR numbers (repeat for several).",
)
parser.add_argument(
"--only-issue",
type=int,
action="append",
default=[],
help="Limit the issue sweep to these issue numbers (repeat for several).",
)
parser.add_argument(
"--ignore-existing-marker",
action="store_true",
help=(
"Re-post on PRs/issues that already carry the heads-up marker. "
"Useful for testing the comment wording on a known PR."
),
)
args = parser.parse_args()
cutoff = args.close_on or (
dt.datetime.now(dt.timezone.utc).date() + dt.timedelta(days=DEFAULT_GRACE_DAYS)
)
kinds: tuple[str, ...]
if args.kind == "pr":
kinds = ("pr",)
elif args.kind == "issue":
kinds = ("issue",)
else:
kinds = ("pr", "issue")
only: dict[str, list[int]] = {}
if args.only_pr:
only["pr"] = args.only_pr
if args.only_issue:
only["issue"] = args.only_issue
# The script must NOT hit the LLM in dry-run if no key is set — we still
# want a useful preview that says "skip-no-llm-key" for items that would
# have been judged. Production runs require OPENAI_API_KEY.
if args.close and not os.environ.get("OPENAI_API_KEY"):
parser.error("OPENAI_API_KEY must be set for --close (real-run) mode.")
run(
repo=args.repo,
close=args.close,
cutoff=cutoff,
model=args.model,
kinds=kinds,
only_numbers=only or None,
skip_marker_check=args.ignore_existing_marker,
)
return 0
if __name__ == "__main__":
sys.exit(main())

30
.github/workflows/_test-unit-base.yml vendored
View file

@ -27,7 +27,7 @@ on:
default: 20
job-timeout-minutes:
description: >-
Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for
Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for
the per-step ceilings on the setup steps below, and 5 for the runner
overhead the job clock charges but no step owns (job init, step
transitions, post-job cleanup). That headroom is what makes the test
@ -36,7 +36,7 @@ on:
arithmetic, so the sum is passed in rather than computed.
required: false
type: number
default: 55
default: 60
max-failures:
description: "Stop after this many failures"
required: false
@ -60,6 +60,9 @@ jobs:
name: Run tests
runs-on: ubuntu-latest
timeout-minutes: ${{ inputs.job-timeout-minutes }}
permissions:
contents: read
pull-requests: read
outputs:
decision: ${{ steps.changes.outputs.decision }}
@ -69,24 +72,27 @@ jobs:
with:
persist-credentials: false
- name: Detect backend-relevant changes
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-backend-changes
uses: ./.github/actions/detect-changes
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
@ -97,6 +103,11 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 8
@ -123,6 +134,13 @@ jobs:
WORKERS: ${{ inputs.workers }}
RERUNS: ${{ inputs.reruns }}
DIST: ${{ inputs.dist }}
# coverage.py's sys.monitoring backend (PEP 669), the cheapest core it has.
# It is only the default from Python 3.14, and these shards run 3.12, so it
# has to be asked for. Coverage refuses it when branch measurement is on
# (`branch_right_left` needs > 3.14.0a5) and falls back to the slow core with
# a `no-sysmon` warning, so turning on `branch = true` here means giving this
# back until the runners move to 3.14.
COVERAGE_CORE: sysmon
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} \
@ -131,7 +149,7 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20 \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
@ -143,7 +161,7 @@ jobs:
--reruns-delay 1 \
--dist="${DIST}" \
--durations=20 \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi

2
.github/workflows/auto_update_price_and_context_window.yml vendored
View file

@ -23,7 +23,7 @@ jobs:
version: "0.10.9"
- name: Update JSON Data
run: |
uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py"
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
- name: Regenerate JSON Schema
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py

22
.github/workflows/check-ui-api-types.yml vendored
View file

@ -67,6 +67,10 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.relevant == 'true'
uses: ./.github/actions/cache-cargo-build
- name: Install backend dependencies
if: steps.changes.outputs.relevant == 'true'
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
@ -79,6 +83,24 @@ jobs:
if: steps.changes.outputs.relevant == 'true'
run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Regenerate the lazy OpenAPI snapshot
if: steps.changes.outputs.relevant == 'true'
run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
- name: Fail if the lazy OpenAPI snapshot is stale
if: steps.changes.outputs.relevant == 'true'
run: |
if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
echo ""
echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
echo "To fix, run from the repo root:"
echo " uv run python -m litellm.proxy._lazy_openapi_snapshot"
echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
exit 1
fi
echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
- name: Set up Node.js
if: steps.changes.outputs.relevant == 'true'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0

9
.github/workflows/ci-coverage.yml vendored
View file

@ -40,3 +40,12 @@ jobs:
run: |
python -m pip install "pyyaml==6.0.3"
python .github/scripts/assert_ci_coverage.py
# The census asks whether a job names a file; this asks whether that job's -k
# then throws it back out. A file both globbed and deselected everywhere runs
# nowhere while counting as covered, which is how the caching suite went unrun.
- name: Assert no -k expression deselects a file from every job that globs it
run: python .github/scripts/assert_ci_coverage.py --slices
- name: Assert .github/workflows/ holds only workflows, correctly named
run: python .github/scripts/assert_workflow_dir_hygiene.py

22
.github/workflows/codspeed.yml vendored
View file

@ -12,6 +12,7 @@ on:
- "uv.lock"
- ".github/workflows/codspeed.yml"
- ".github/actions/setup-uv-with-retries/**"
- ".github/actions/cache-cargo-build/**"
pull_request:
branches:
- main
@ -23,6 +24,7 @@ on:
- "uv.lock"
- ".github/workflows/codspeed.yml"
- ".github/actions/setup-uv-with-retries/**"
- ".github/actions/cache-cargo-build/**"
# Allow CodSpeed to trigger backtest performance analysis
# in order to generate initial data
workflow_dispatch:
@ -55,6 +57,26 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
# Build the wheel and resolve every dependency outside the CodSpeed
# runner: the same maturin build took 42 minutes inside `codspeed run`
# versus under 3 minutes as a plain step (LIT-6183)
- name: Build environment
run: >
env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
uv run --frozen --no-default-groups
--with pytest==8.3.5
--with pytest-codspeed==4.3.0
--with "mcp>=1.26.0,<2.0"
--with "a2a-sdk>=1.1.0,<2.0"
pytest
-p pytest_codspeed.plugin
tests/benchmarks/
--codspeed
--collect-only -q
- name: Run benchmarks
uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
with:

237
.github/workflows/e2e_record_replay.yml vendored Normal file
View file

@ -0,0 +1,237 @@
name: "E2E Record and Replay"
on:
schedule:
- cron: "0 8 * * 6"
- cron: "0 8 * * 1-5"
workflow_dispatch:
inputs:
mode:
description: "record (hits real providers and publishes a fresh bundle) or replay (bundle only, zero provider egress)"
type: choice
options:
- record
- replay
default: record
permissions:
contents: read
jobs:
record:
name: "Record the e2e suite against real providers"
if: >-
(github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
(github.event.schedule == '0 8 * * 6' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record'))
runs-on: ubuntu-latest
timeout-minutes: 45
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U llmproxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-e2e-record-replay
LITELLM_LOCAL_MODEL_COST_MAP: "True"
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
fi
sleep 2
done
echo "proxy never became live"
tail -n 100 proxy.log
exit 1
- name: Record the replayable e2e lane
env:
E2E_FIXTURE_MODE: record
run: |
uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
- name: Pack the fixture bundle
run: |
.github/scripts/e2e_pack_fixture_bundle.sh tests/e2e/.fixtures "${RUNNER_TEMP}/bundle/e2e-fixtures.tar.gz"
- name: Publish the fixture bundle
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: e2e-fixtures-bundle
path: |
${{ runner.temp }}/bundle/e2e-fixtures.tar.gz
${{ runner.temp }}/bundle/e2e-fixtures.tar.gz.sha256
if-no-files-found: error
retention-days: 30
- name: Show proxy log on failure
if: failure()
run: tail -n 300 proxy.log
replay:
name: "Replay the e2e suite from the pinned bundle with zero egress"
if: >-
(github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
(github.event.schedule == '0 8 * * 1-5' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'replay'))
runs-on: ubuntu-latest
timeout-minutes: 45
permissions:
contents: read
actions: read
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U llmproxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-e2e-record-replay
LITELLM_LOCAL_MODEL_COST_MAP: "True"
GH_TOKEN: ${{ github.token }}
OPENAI_API_KEY: sk-replay-must-never-reach-a-provider
ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Fetch the pinned fixture bundle by digest
env:
BASE_BRANCH: ${{ github.ref_name }}
run: |
.github/scripts/e2e_fetch_fixture_bundle.sh \
"${GITHUB_REPOSITORY}" \
e2e-fixtures-bundle \
"${BASE_BRANCH}" \
tests/e2e/.fixtures
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
fi
sleep 2
done
echo "proxy never became live"
tail -n 100 proxy.log
exit 1
- name: Start the egress sentinel
run: |
# shellcheck disable=SC2024 # the log redirect is deliberately the runner user's, so a later non-sudo cat can read it
sudo python3 .github/scripts/e2e_egress_sentinel.py serve \
--host api.openai.com \
--host api.anthropic.com \
--hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \
--ready-file "${RUNNER_TEMP}/egress-ready" \
--pid-file "${RUNNER_TEMP}/egress.pid" \
> "${RUNNER_TEMP}/egress-sentinel.log" 2>&1 &
for _ in $(seq 1 30); do
if [[ -f "${RUNNER_TEMP}/egress-ready" ]]; then
cat "${RUNNER_TEMP}/egress-sentinel.log"
exit 0
fi
sleep 1
done
echo "egress sentinel never became ready"
cat "${RUNNER_TEMP}/egress-sentinel.log"
exit 1
- name: Replay the replayable e2e lane
env:
E2E_FIXTURE_MODE: replay
run: |
uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
- name: Stop the egress sentinel and assert zero provider egress
if: always()
run: |
if [[ -f "${RUNNER_TEMP}/egress.pid" ]]; then
sudo kill -TERM "$(cat "${RUNNER_TEMP}/egress.pid")" 2>/dev/null || true
sleep 2
fi
python3 .github/scripts/e2e_egress_sentinel.py assert-empty --hits-file "${RUNNER_TEMP}/egress-hits.jsonl"
- name: Show proxy log on failure
if: failure()
run: tail -n 300 proxy.log

33
.github/workflows/image-scan.yml vendored
View file

@ -17,10 +17,14 @@ on:
- backend/Dockerfile
- backend/main.py
- docker/component_entrypoint.sh
- docker/entrypoint.sh
- litellm/proxy/prisma_migration.py
- litellm-proxy-extras/**
- tests/proxy_migration_tests/**
- uv.lock
- ui/litellm-dashboard/package-lock.json
- ui/Dockerfile
- ui/nginx.conf
- .github/workflows/image-scan.yml
schedule:
- cron: "41 6 * * *"
@ -183,6 +187,35 @@ jobs:
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
ui-image:
name: ui-image
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build UI image
run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
env:
LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
backend-image:
name: backend-image
runs-on: ubuntu-latest

13
.github/workflows/mutation-test.yml vendored
View file

@ -53,6 +53,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
@ -84,11 +87,20 @@ jobs:
run: |
uv pip uninstall pytest-retry || true
# Ends before the job's own deadline so a run that outlasts the budget is
# still followed by the report and upload steps. mutmut saves after every
# mutant result, to mutants/<source path>.meta, so an interrupted run
# still scores the mutants it finished and export-cicd-stats can read
# them; a cancelled job skips those steps and publishes nothing at all.
- name: Run mutmut
timeout-minutes: 300
env:
# Make the mutants/ sandbox win over site-packages on sys.path so the
# trampolined files are imported instead of the installed copy.
PYTHONPATH: ${{ github.workspace }}/mutants
# Without this mutmut finds no covered lines and generates 0 mutants.
# See the file itself for why.
COVERAGE_RCFILE: ${{ github.workspace }}/.github/mutmut-coverage.rc
run: |
set -o pipefail
mkdir -p mutants
@ -127,6 +139,7 @@ jobs:
mutmut-run.log
mutants/mutmut-stats.json
mutants/mutmut-cicd-stats.json
mutants/**/*.meta
mutants/litellm/proxy/management_endpoints/**/*.py
if-no-files-found: warn
retention-days: 14

3
.github/workflows/publish-basedpyright-base-counts.yml vendored
View file

@ -43,6 +43,9 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries

68
.github/workflows/sync-together-ai-models.yml vendored Normal file
View file

@ -0,0 +1,68 @@
name: Sync Together AI model registry
on:
schedule:
- cron: "30 6 * * *"
workflow_dispatch:
permissions:
contents: write
pull-requests: write
jobs:
sync_together_ai_models:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: litellm_internal_staging
persist-credentials: false
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Look for an already-open sync PR
id: existing
run: |
open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
--jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
if [ -n "$open_pr" ]; then
echo "An open sync PR already exists on branch $open_pr; skipping this run."
fi
env:
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
- name: Run the sync
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
env:
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
- name: Regenerate the JSON schema
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python ci_cd/generate_model_prices_schema.py
- name: Create a pull request when the registry changed
if: steps.existing.outputs.open_pr == ''
run: |
if git diff --quiet; then
echo "Registry already in sync; no PR needed."
exit 0
fi
branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -b "$branch"
git add model_prices_and_context_window.json \
litellm/model_prices_and_context_window_backup.json \
model_prices_and_context_window.schema.json
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
gh auth setup-git
git push origin "$branch"
gh pr create --title "feat(models): sync together_ai model registry" \
--body-file "$RUNNER_TEMP/pr_body.md" \
--head "$branch" \
--base litellm_internal_staging
env:
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}

6
.github/workflows/test-code-quality.yml vendored
View file

@ -56,6 +56,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: uv sync --frozen --all-groups --all-extras
@ -128,6 +131,9 @@ jobs:
- name: check_e2e_no_raw_requests
run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
- name: check_migrations_no_data_rewrites
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
- name: memory_test
run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py

50
.github/workflows/test-linting.yml vendored
View file

@ -24,6 +24,7 @@ jobs:
# re-running basedpyright over the merge-base tree.
permissions:
contents: read
pull-requests: read
actions: read
steps:
@ -37,7 +38,12 @@ jobs:
clean: true
persist-credentials: false
- name: Detect relevant changes
id: changes
uses: ./.github/actions/detect-changes
- name: Fetch gate base (merge-base with target branch)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
@ -50,39 +56,62 @@ jobs:
echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-lint-
- name: Clean Python cache
if: steps.changes.outputs.decision != 'skip'
run: |
find . -type d -name "__pycache__" -exec rm -rf {} + || true
find . -name "*.pyc" -delete || true
- name: Check uv.lock is up to date
if: steps.changes.outputs.decision != 'skip'
run: |
uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
uv sync --frozen --group proxy-dev --group e2e-dev
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-prisma-binaries
# basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma)
# only after `prisma generate` writes prisma/client.py et al. Without this the
# DB wrappers typed against the generated client would degrade to Unknown.
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Check ruff format
if: steps.changes.outputs.decision != 'skip'
run: |
git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
@ -92,6 +121,7 @@ jobs:
xargs uv run --no-sync ruff format --check --exclude '/enterprise/' < "$RUNNER_TEMP/ruff_format_files.txt"
- name: Debug - Check file state
if: steps.changes.outputs.decision != 'skip'
run: |
echo "Current branch:"
git branch --show-current
@ -101,30 +131,46 @@ jobs:
head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
- name: Run Ruff linting
if: steps.changes.outputs.decision != 'skip'
run: |
cd litellm
uv run --no-sync ruff check .
cd ..
- name: Run Ruff linting (test tree)
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync ruff check --config ruff-tests.toml tests
- name: Check strict-rule budget (delta vs base)
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
- name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
- name: Check test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs base)
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA"
- name: Print OpenAI version
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
- name: Check basedpyright budget (delta vs base)
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
run: |
uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA"
- name: Check tests/e2e basedpyright (zero errors)
if: steps.changes.outputs.decision != 'skip'
run: |
if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
uv run --no-sync basedpyright tests/e2e
@ -133,12 +179,14 @@ jobs:
fi
- name: Check for circular imports
if: steps.changes.outputs.decision != 'skip'
run: |
cd litellm
uv run --no-sync python ../tests/documentation_tests/test_circular_imports.py
cd ..
- name: Check import safety
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
@ -200,7 +248,7 @@ jobs:
- name: Run secret scan test
run: |
uv run --no-project --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v
uv run --no-project --with 'pytest==9.0.2' pytest tests/code_coverage_tests/test_no_hardcoded_secrets.py -v
- name: Run ggshield secret scan
env:

10
.github/workflows/test-litellm-ui-build.yml vendored
View file

@ -1,6 +1,7 @@
name: UI Build Check
permissions:
contents: read
pull-requests: read
on:
pull_request:
@ -28,7 +29,14 @@ jobs:
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
uses: ./.github/actions/detect-changes
with:
category: ui
- name: Setup Node.js
if: steps.changes.outputs.decision != 'skip'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
@ -36,7 +44,9 @@ jobs:
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: npm ci
- name: Build
if: steps.changes.outputs.decision != 'skip'
run: npm run build

56
.github/workflows/test-litellm-ui-unit.yml vendored
View file

@ -1,6 +1,7 @@
name: UI Unit Tests
permissions:
contents: read
pull-requests: read
on:
pull_request:
@ -32,7 +33,14 @@ jobs:
fetch-depth: 1
persist-credentials: false
- name: Detect relevant changes
id: changes
uses: ./.github/actions/detect-changes
with:
category: ui
- name: Setup Node.js
if: steps.changes.outputs.decision != 'skip'
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
@ -40,36 +48,50 @@ jobs:
cache-dependency-path: ui/litellm-dashboard/package-lock.json
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: npm ci
- name: Run UI type tests (Vitest)
if: steps.changes.outputs.decision != 'skip'
env:
CI: "true"
run: npm run test:types
- name: Run UI unit tests (Vitest)
if: steps.changes.outputs.decision != 'skip'
env:
CI: "true"
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
if [ -n "$BASE_SHA" ]; then
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
changed_files=()
while IFS= read -r f; do
changed_files+=("$f")
done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
if [ ${#changed_files[@]} -eq 0 ]; then
echo "No UI files changed in this PR; skipping unit tests."
exit 0
fi
echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
--pool forks --poolOptions.forks.maxForks=14
else
full_suite() { npm run test -- --run --pool forks --poolOptions.forks.maxForks=14; }
if [ -z "$BASE_SHA" ]; then
echo "Push to $GITHUB_REF_NAME: running the full suite"
npm run test -- --run --pool forks --poolOptions.forks.maxForks=14
full_suite
exit 0
fi
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
changed_files=()
while IFS= read -r f; do
changed_files+=("$f")
done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
if [ ${#changed_files[@]} -eq 0 ]; then
echo "No UI files changed in this PR; skipping unit tests."
exit 0
fi
scope=$(printf '%s\n' "${changed_files[@]}" | bash "$GITHUB_WORKSPACE/.github/scripts/select_ui_test_scope.sh")
if [ "$scope" != related ]; then
echo "Pull request: ${#changed_files[@]} changed UI files reach outside src/, so related would miss their dependents; running the full suite"
full_suite
exit 0
fi
echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
--pool forks --poolOptions.forks.maxForks=14

15
.github/workflows/test-mcp.yml vendored
View file

@ -10,6 +10,7 @@ on:
permissions:
contents: read
pull-requests: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
@ -25,26 +26,38 @@ jobs:
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
uses: ./.github/actions/detect-changes
- name: Thank You Message
run: |
echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
uv lock --check
.github/scripts/uv_sync_with_retries.sh --frozen --group proxy-dev --extra proxy --extra semantic-router
- name: Run MCP tests
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5

0
.github/workflows/test-model-map.yaml → .github/workflows/test-model-map.yml vendored
View file

145
.github/workflows/test-postgres.yml vendored Normal file
View file

@ -0,0 +1,145 @@
name: "Postgres Tests"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
postgres:
name: ${{ matrix.shard }}
runs-on: ubuntu-latest
timeout-minutes: ${{ matrix.job-timeout-minutes }}
permissions:
contents: read
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 10
strategy:
fail-fast: false
matrix:
include:
- shard: proxy-behavior
test-path: "tests/proxy_behavior"
seed: db-push
workers: 0
timeout-minutes: 25
job-timeout-minutes: 50
- shard: proxy-security
test-path: "tests/proxy_security_tests"
seed: db-push
workers: 0
timeout-minutes: 15
job-timeout-minutes: 40
- shard: schema-migration
test-path: "tests/proxy_migration_tests"
seed: none
workers: 0
timeout-minutes: 20
job-timeout-minutes: 45
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
timeout-minutes: 3
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-changes
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-postgres-
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 12
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Seed database schema
if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
timeout-minutes: 10
run: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- name: Run tests
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: ${{ matrix.timeout-minutes }}
env:
TEST_PATH: ${{ matrix.test-path }}
WORKERS: ${{ matrix.workers }}
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
else
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
fi

54
.github/workflows/test-terraform-modules.yml vendored Normal file
View file

@ -0,0 +1,54 @@
name: Terraform Modules
on:
push:
paths:
- "terraform/litellm/aws/**"
- ".github/workflows/test-terraform-modules.yml"
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
paths:
- "terraform/litellm/aws/**"
- ".github/workflows/test-terraform-modules.yml"
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
aws-module:
name: fmt, validate, test (aws)
runs-on: ubuntu-latest
timeout-minutes: 15
defaults:
run:
working-directory: terraform/litellm/aws
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
with:
terraform_version: 1.13.3
terraform_wrapper: false
- name: fmt
run: terraform fmt -recursive -check -diff
- name: init
run: terraform init -backend=false -input=false
- name: validate
run: terraform validate
# Plan-only, mock_provider-backed: no AWS credentials, no API calls.
- name: test
run: terraform test

5
.github/workflows/test-terraform-provider.yml vendored
View file

@ -88,6 +88,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
@ -111,4 +114,4 @@ jobs:
- name: Audit provider endpoints against the schema
working-directory: terraform/provider
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json"
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json" -coverage-allowlist ./tools/endpointaudit/coverage_allowlist.txt

31
.github/workflows/test-unit-core-utils.yml vendored
View file

@ -1,31 +0,0 @@
name: "Unit Tests: Core Utilities"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
core-utils:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/litellm_core_utils"
workers: 2
reruns: 1
artifact-name: core-utils

19
.github/workflows/test-unit-documentation.yml vendored
View file

@ -23,34 +23,41 @@ jobs:
documentation:
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
pull-requests: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
uses: ./.github/actions/detect-changes
- name: Checkout litellm-docs into docs/my-website (for documentation_tests)
if: steps.changes.outputs.decision != 'skip'
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
repository: BerriAI/litellm-docs
path: docs/my-website
persist-credentials: false
- name: Detect backend-relevant changes
id: changes
uses: ./.github/actions/detect-backend-changes
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
@ -60,6 +67,10 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |

35
.github/workflows/test-unit-enterprise-routing.yml vendored
View file

@ -1,35 +0,0 @@
name: "Unit Tests: Enterprise, Google GenAI & Routing"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
enterprise-routing:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: >-
tests/test_litellm/enterprise
tests/test_litellm/google_genai
tests/test_litellm/router_utils
tests/test_litellm/router_strategy
workers: 2
reruns: 2
artifact-name: enterprise-routing

31
.github/workflows/test-unit-integrations.yml vendored
View file

@ -1,31 +0,0 @@
name: "Unit Tests: Integrations (Callbacks & Logging)"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
integrations:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/integrations"
workers: 2
reruns: 3
artifact-name: integrations

47
.github/workflows/test-unit-llm-providers.yml vendored
View file

@ -1,47 +0,0 @@
name: "Unit Tests: LLM Provider Transformations"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
vertex-ai:
name: Vertex AI
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/llms/vertex_ai"
workers: 1
reruns: 2
artifact-name: llm-vertex-ai
other-providers:
name: All Other Providers
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
workers: 2
reruns: 2
artifact-name: llm-other-providers

53
.github/workflows/test-unit-misc.yml vendored
View file

@ -1,53 +0,0 @@
name: "Unit Tests: MCP, Secrets, Containers & Misc"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
misc:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: >-
tests/test_litellm/batches
tests/test_litellm/secret_managers
tests/test_litellm/a2a_protocol
tests/test_litellm/anthropic_interface
tests/test_litellm/completion_extras
tests/test_litellm/compression
tests/test_litellm/containers
tests/test_litellm/experimental_mcp_client
tests/test_litellm/models
tests/test_litellm/repositories
tests/test_litellm/images
tests/test_litellm/interactions
tests/test_litellm/ocr
tests/test_litellm/passthrough
tests/test_litellm/rag
tests/test_litellm/realtime_api
tests/test_litellm/rerank_api
tests/test_litellm/sandbox
tests/test_litellm/test_router
tests/test_litellm/vector_stores
tests/test_litellm/videos
tests/test_litellm/test_*.py
workers: 2
reruns: 2
artifact-name: misc

31
.github/workflows/test-unit-proxy-auth.yml vendored
View file

@ -1,31 +0,0 @@
name: "Unit Tests: Proxy Auth & Key Management"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
proxy-auth:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client"
workers: 2
reruns: 2
artifact-name: proxy-auth

38
.github/workflows/test-unit-proxy-db.yml vendored
View file

@ -42,11 +42,10 @@ concurrency:
# pinning the whole file to one worker (the default --dist=loadscope
# behavior for single-file targets).
jobs:
# Fast guard — fails the workflow if a test_*.py file under
# tests/proxy_unit_tests/ is not referenced by any matrix entry below.
# The semantic-shard design (no catch-all "remaining" bucket) relies on
# every test file being explicitly assigned; this guard prevents a new
# file from silently dropping out of CI.
# Fast guard — fails the workflow when a test directory or file inside a sharded
# tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
# so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list
# and reads the same test-path keys the coverage census does.
assert-shard-coverage:
runs-on: ubuntu-latest
timeout-minutes: 2
@ -56,31 +55,8 @@ jobs:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Assert every test_*.py is in a matrix shard
run: |
python3 - <<'PY'
import pathlib, sys, yaml
wf = yaml.safe_load(open(".github/workflows/test-unit-proxy-db.yml"))
matrix = wf["jobs"]["proxy-db"]["strategy"]["matrix"]["include"]
referenced = set()
for entry in matrix:
for token in entry["test-path"].split():
if token.startswith("tests/proxy_unit_tests/"):
referenced.add(pathlib.PurePosixPath(token).name)
actual = {p.name for p in pathlib.Path("tests/proxy_unit_tests").iterdir()
if p.name.startswith("test_") and (p.suffix == ".py" or p.is_dir())
and p.name != "test_configs"}
orphans = sorted(actual - referenced)
if orphans:
print("ERROR: the following files/dirs under tests/proxy_unit_tests/")
print(" are not assigned to any shard in test-unit-proxy-db.yml:")
for o in orphans:
print(f" - {o}")
print()
print("Add each to whichever semantic shard it belongs to.")
sys.exit(1)
print(f"OK: all {len(actual)} files assigned to a shard.")
PY
- name: Assert every test directory and file is claimed by a shard
run: python3 .github/scripts/assert_ci_coverage.py --shards
proxy-db:
needs: assert-shard-coverage
@ -135,8 +111,6 @@ jobs:
test-path: >-
tests/proxy_unit_tests/test_proxy_server.py
tests/proxy_unit_tests/test_proxy_server_keys.py
tests/proxy_unit_tests/test_proxy_server_caching.py
tests/proxy_unit_tests/test_proxy_server_langfuse.py
tests/proxy_unit_tests/test_proxy_server_spend.py
tests/proxy_unit_tests/test_aproxy_startup.py
workers: 4

80
.github/workflows/test-unit-proxy-endpoints.yml vendored
View file

@ -1,80 +0,0 @@
name: "Unit Tests: Proxy API Endpoints"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
proxy-endpoints:
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: >-
tests/test_litellm/proxy/analytics_endpoints
tests/test_litellm/proxy/management_endpoints
tests/test_litellm/proxy/memory
tests/test_litellm/proxy/guardrails
tests/test_litellm/proxy/management_helpers
tests/test_litellm/proxy/anthropic_endpoints
tests/test_litellm/proxy/google_endpoints
tests/test_litellm/proxy/openai_files_endpoint
tests/test_litellm/proxy/batches_endpoints
tests/test_litellm/proxy/fine_tuning_endpoints
tests/test_litellm/proxy/vector_store_files_endpoints
tests/test_litellm/proxy/video_endpoints
tests/test_litellm/proxy/response_api_endpoints
tests/test_litellm/proxy/image_endpoints
tests/test_litellm/proxy/vector_store_endpoints
tests/test_litellm/proxy/agent_endpoints
tests/test_litellm/proxy/a2a
tests/test_litellm/proxy/credential_endpoints
tests/test_litellm/proxy/discovery_endpoints
tests/test_litellm/proxy/health_endpoints
tests/test_litellm/proxy/shutdown
tests/test_litellm/proxy/public_endpoints
tests/test_litellm/proxy/prompts
tests/test_litellm/proxy/rag_endpoints
tests/test_litellm/proxy/realtime_endpoints
tests/test_litellm/proxy/ui_crud_endpoints
tests/test_litellm/proxy/config_resolvers
tests/test_litellm/proxy/utils
workers: 2
reruns: 2
artifact-name: proxy-endpoints
# Behavior-pinning tests for litellm/proxy/proxy_server.py. Owns its
# own job (not a path on the proxy-endpoints job above) so its budget
# is independent and its coverage artifact is uploaded separately.
# See: https://www.notion.so/36c43b8acdab81ee845fd5365128a2fc
proxy-server:
permissions:
contents: read
id-token: write
pull-requests: write
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: tests/test_litellm/proxy/proxy_server
workers: 4
reruns: 2
timeout-minutes: 60
job-timeout-minutes: 95
artifact-name: proxy-server

42
.github/workflows/test-unit-proxy-infra.yml vendored
View file

@ -1,42 +0,0 @@
name: "Unit Tests: Proxy Infrastructure"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
proxy-infra:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: >-
tests/test_litellm/proxy/db
tests/test_litellm/proxy/middleware
tests/test_litellm/proxy/spend_tracking
tests/test_litellm/proxy/pass_through_endpoints
tests/test_litellm/proxy/_experimental
tests/test_litellm/proxy/experimental
tests/test_litellm/proxy/common_utils
tests/test_litellm/proxy/enterprise_billing
tests/test_litellm/proxy/types_utils
tests/test_litellm/proxy/logging_endpoints
tests/test_litellm/proxy/test_*.py
workers: 2
reruns: 2
artifact-name: proxy-infra

106
.github/workflows/test-unit-proxy-legacy.yml vendored
View file

@ -1,106 +0,0 @@
name: "Unit Tests: Proxy Legacy Tests"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
test-group:
- name: "auth-and-jwt"
path: "tests/proxy_unit_tests/test_[a-j]*.py"
- name: "key-generation"
path: "tests/proxy_unit_tests/test_[k-o]*.py"
- name: "proxy-config"
path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
- name: "proxy-server"
path: "tests/proxy_unit_tests/test_proxy_server.py"
- name: "proxy-server-extras"
path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
- name: "proxy-utils"
path: "tests/proxy_unit_tests/test_proxy_utils.py"
- name: "proxy-token-counter"
path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
- name: "proxy-response-and-misc"
path: "tests/proxy_unit_tests/test_[r-t]*.py"
- name: "proxy-user-auth-and-spend"
path: "tests/proxy_unit_tests/test_[u-z]*.py"
name: ${{ matrix.test-group.name }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Detect backend-relevant changes
id: changes
uses: ./.github/actions/detect-backend-changes
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Run tests - ${{ matrix.test-group.name }}
if: steps.changes.outputs.decision != 'skip'
env:
TEST_PATH: ${{ matrix.test-group.path }}
run: |
uv run --no-sync pytest ${TEST_PATH} \
--tb=short -vv \
--maxfail=10 \
-n 2 \
--reruns 1 \
--reruns-delay 1 \
--dist=loadscope \
--durations=20

31
.github/workflows/test-unit-responses-caching-types.yml vendored
View file

@ -1,31 +0,0 @@
name: "Unit Tests: Responses, Caching & Types"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
permissions:
contents: read
id-token: write
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
responses-caching-types:
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
workers: 2
reruns: 2
artifact-name: responses-caching-types

251
.github/workflows/test-unit.yml vendored Normal file
View file

@ -0,0 +1,251 @@
name: "Unit Tests"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One caller for every tests/test_litellm shard, replacing the nine thin workflow
# files that each wrapped a single call to _test-unit-base.yml. Adding a shard is
# now one matrix entry rather than a new file.
#
# `name` is the shard id and nothing else, so each check reports as
# "<shard> / Run tests" exactly as it did when the shard had its own file. Those
# strings are the branch ruleset's required contexts, so they are load-bearing:
# renaming an entry renames a required check and the ruleset stops matching it.
#
# Every entry states its timeouts even when they equal the base workflow's
# defaults. An absent matrix key renders as an empty string, which is not a
# number, so a partially-specified entry would fail the call rather than fall
# back to the default.
#
# tests/proxy_unit_tests keeps its own caller (test-unit-proxy-db.yml): it is
# already a matrix and carries a shard-coverage guard that reads that file by
# name. Folding it in here is a follow-up, together with generalising that guard
# into assert_ci_coverage.py.
jobs:
unit:
name: ${{ matrix.shard }}
permissions:
contents: read
id-token: write
pull-requests: write
strategy:
fail-fast: false
matrix:
include:
- shard: core-utils
artifact-name: core-utils
test-path: "tests/test_litellm/litellm_core_utils"
workers: 2
reruns: 1
timeout-minutes: 20
job-timeout-minutes: 60
- shard: enterprise-routing
artifact-name: enterprise-routing
test-path: >-
tests/test_litellm/enterprise
tests/test_litellm/google_genai
tests/test_litellm/router_utils
tests/test_litellm/router_strategy
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: integrations
artifact-name: integrations
test-path: "tests/test_litellm/integrations"
workers: 2
reruns: 3
timeout-minutes: 20
job-timeout-minutes: 60
- shard: Vertex AI
artifact-name: llm-vertex-ai
test-path: "tests/test_litellm/llms/vertex_ai"
workers: 1
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: All Other Providers
artifact-name: llm-other-providers
test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: misc
artifact-name: misc
test-path: >-
tests/test_litellm/batches
tests/test_litellm/secret_managers
tests/test_litellm/a2a_protocol
tests/test_litellm/anthropic_interface
tests/test_litellm/completion_extras
tests/test_litellm/compression
tests/test_litellm/containers
tests/test_litellm/experimental_mcp_client
tests/test_litellm/models
tests/test_litellm/repositories
tests/test_litellm/images
tests/test_litellm/interactions
tests/test_litellm/ocr
tests/test_litellm/passthrough
tests/test_litellm/rag
tests/test_litellm/realtime_api
tests/test_litellm/rerank_api
tests/test_litellm/rust_bridge
tests/test_litellm/sandbox
tests/test_litellm/test_router
tests/test_litellm/vector_stores
tests/test_litellm/videos
tests/test_litellm/test_*.py
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-auth
artifact-name: proxy-auth
test-path: >-
tests/test_litellm/proxy/auth
tests/test_litellm/proxy/hooks
tests/test_litellm/proxy/policy_engine
tests/test_litellm/proxy/client
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-endpoints
artifact-name: proxy-endpoints
test-path: >-
tests/test_litellm/proxy/analytics_endpoints
tests/test_litellm/proxy/management_endpoints
tests/test_litellm/proxy/list_api
tests/test_litellm/proxy/memory
tests/test_litellm/proxy/guardrails
tests/test_litellm/proxy/management_helpers
tests/test_litellm/proxy/anthropic_endpoints
tests/test_litellm/proxy/google_endpoints
tests/test_litellm/proxy/openai_files_endpoint
tests/test_litellm/proxy/batches_endpoints
tests/test_litellm/proxy/fine_tuning_endpoints
tests/test_litellm/proxy/vector_store_files_endpoints
tests/test_litellm/proxy/video_endpoints
tests/test_litellm/proxy/response_api_endpoints
tests/test_litellm/proxy/image_endpoints
tests/test_litellm/proxy/ocr_endpoints
tests/test_litellm/proxy/vector_store_endpoints
tests/test_litellm/proxy/agent_endpoints
tests/test_litellm/proxy/a2a
tests/test_litellm/proxy/credential_endpoints
tests/test_litellm/proxy/discovery_endpoints
tests/test_litellm/proxy/health_endpoints
tests/test_litellm/proxy/shutdown
tests/test_litellm/proxy/public_endpoints
tests/test_litellm/proxy/prompts
tests/test_litellm/proxy/rag_endpoints
tests/test_litellm/proxy/rerank_endpoints
tests/test_litellm/proxy/realtime_endpoints
tests/test_litellm/proxy/ui_crud_endpoints
tests/test_litellm/proxy/config_resolvers
tests/test_litellm/proxy/utils
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-server
artifact-name: proxy-server
test-path: "tests/test_litellm/proxy/proxy_server"
workers: 4
reruns: 2
timeout-minutes: 60
job-timeout-minutes: 100
- shard: proxy-infra
artifact-name: proxy-infra
test-path: >-
tests/test_litellm/proxy/db
tests/test_litellm/proxy/middleware
tests/test_litellm/proxy/spend_tracking
tests/test_litellm/proxy/pass_through_endpoints
tests/test_litellm/proxy/_experimental
tests/test_litellm/proxy/experimental
tests/test_litellm/proxy/common_utils
tests/test_litellm/proxy/enterprise_billing
tests/test_litellm/proxy/types_utils
tests/test_litellm/proxy/logging_endpoints
tests/test_litellm/proxy/test_*.py
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: caching-local
artifact-name: caching-local
test-path: >-
tests/local_testing/test_cache_preset_key.py
tests/local_testing/test_caching_handler.py
tests/local_testing/test_prompt_caching.py
tests/local_testing/test_responses_stream_cache_keys.py
tests/local_testing/test_unit_test_caching.py
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: proxy-extras
artifact-name: proxy-extras
test-path: "tests/litellm-proxy-extras"
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: enterprise-package
artifact-name: enterprise-package
test-path: "tests/enterprise"
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: responses-caching-types
artifact-name: responses-caching-types
test-path: >-
tests/test_litellm/responses
tests/test_litellm/caching
tests/test_litellm/types
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}
workers: ${{ matrix.workers }}
reruns: ${{ matrix.reruns }}
timeout-minutes: ${{ matrix.timeout-minutes }}
job-timeout-minutes: ${{ matrix.job-timeout-minutes }}
artifact-name: ${{ matrix.artifact-name }}

92
.github/workflows/triage_rollout_heads_up.yml vendored
View file

@ -1,92 +0,0 @@
name: Agent Shin — rollout heads-up (one-shot)
# Fires the 7-day heads-up comment on every open external PR/issue that the
# new triage bot would auto-close. The real sweep is a deliberate one-shot:
# trigger it at rollout via a manual `workflow_dispatch` with `dry_run=false`.
# The script is idempotent (skips items that already carry the
# `<!-- agent-shin:rollout-heads-up -->` marker), so a re-run is harmless.
#
# The automatic push trigger runs DRY-RUN only, so merging the script to
# `litellm_internal_staging` never posts a comment; it just confirms the
# workflow is wired up. Posting real comments requires the manual dispatch,
# which is also the only trigger that exposes `OPENAI_API_KEY`. The heads-up
# is intentionally NOT gated on `AGENT_SHIN_ENABLED`: it has to warn
# contributors while that flag is still off, ahead of the flip that turns on
# auto-closing.
#
# The workflow is a thin shell over `.github/scripts/triage_rollout_heads_up.py`.
# Dry-run vs. real run differ in EXACTLY one CLI flag (`--close`), added only
# on a manual dispatch with `dry_run=false`.
on:
push:
branches:
- litellm_internal_staging
paths:
# The presence of this script on staging IS the rollout merge marker.
# Editing the file later would re-fire the workflow; that's safe because
# the script skips PRs/issues that already have the heads-up marker.
- ".github/scripts/triage_rollout_heads_up.py"
workflow_dispatch:
inputs:
dry_run:
description: "Dry run (true = preview only, false = actually post comments)."
required: false
default: "true"
type: choice
options:
- "true"
- "false"
permissions:
contents: read
issues: write
pull-requests: write
jobs:
heads-up:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- name: Checkout triage scripts
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Install LLM client
run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
- name: Run heads-up sweep
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# Only the manual dispatch (the real-run trigger) needs the LLM key.
# The automatic push trigger runs dry-run and never posts, so it gets
# no key. Mirrors the sibling triage workflows, which expose the key
# only on an enabled/dispatched run rather than unconditionally.
OPENAI_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.OPENAI_API_KEY || '' }}
OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
# The real run is a deliberate manual dispatch with dry_run=false.
# Use the EXACT "false" comparison so any unexpected input value
# fail-closes to dry-run (mirrors the AGENT_SHIN_ENABLED pattern in
# the sibling workflows). The automatic push trigger always stays
# dry-run, so merging the script never posts.
DRY_RUN_INPUT: ${{ github.event.inputs.dry_run }}
run: |
set -euo pipefail
ARGS=(--repo "${{ github.repository }}")
if [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${DRY_RUN_INPUT:-true}" = "false" ]; then
ARGS+=(--close)
echo "::notice::Manual rollout dispatch with dry_run=false -> heads-up comments WILL be posted."
elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ]; then
echo "::notice::Manual dispatch in dry-run mode -> previewing only, no comments will be posted."
else
echo "::notice::Automatic push trigger -> dry-run preview only. Fire the real rollout sweep with a manual workflow_dispatch (dry_run=false)."
fi
python3 .github/scripts/triage_rollout_heads_up.py "${ARGS[@]}"

3
.github/workflows/weekly_load_anomaly.yml vendored
View file

@ -47,6 +47,9 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy

4
.gitignore vendored
View file

@ -1,9 +1,13 @@
.python-version
.venv
tests/e2e/.fixtures/
.venv-typecheck
.venv_policy_test
.venv-mutmut
mutants/
.env
.claude
CLAUDE.local.md
.newenv
newenv/*
litellm/proxy/myenv/*

16
CLAUDE.md
View file

@ -29,7 +29,7 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions
When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions
When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively
@ -37,13 +37,14 @@ If you're resolving a linear ticket, in the "## Linear ticket" section of the PR
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
- don't use emojis
- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
- don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
- unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
- don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
@ -53,6 +54,8 @@ When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-bud
`make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone budget gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
@ -63,6 +66,8 @@ Commit and push your work when you're done without asking
When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch
@ -77,13 +82,16 @@ Do not put names of customers or customer company names in code, PR descriptions
CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
- Composition over inheritance
- Never-nester: early returns over deep nesting
- Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
- Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>` explaining why
- Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
- Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: <reason>`
- Use dependency injection
- Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
- Use tagged unions + match

18
CONTRIBUTING.md
View file

@ -13,8 +13,8 @@ Here are the core requirements for any PR submitted to LiteLLM:
- [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
- [ ] **Ensure your PR passes all checks**:
- [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
- [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
- [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/test_litellm/<your_test_file>.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally
#### UI PRs
@ -71,8 +71,8 @@ make format
# Run all linting checks (matches CI exactly)
make lint
# Run unit tests to ensure nothing is broken
make test-unit
# Run the tests covering your change (CI runs the full suite)
uv run pytest tests/test_litellm/<your_test_file>.py -v
# Commit your changes (must follow Conventional Commits — see above)
git add .
@ -123,12 +123,13 @@ def test_your_feature():
### Running Unit Tests
Run all unit tests (uses parallel execution for speed):
Run the tests covering your change:
```bash
make test-unit
uv run pytest tests/test_litellm/test_your_file.py -v
```
`tests/test_litellm` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that.
If you're running broader test suites, proxy tests, or anything that touches PostgreSQL-backed fixtures/plugins, install the full local test environment first:
```bash
@ -137,11 +138,6 @@ make install-test-deps
This syncs the locked test environment used across the repo, including `psycopg` v3 plus `psycopg-binary` (used by `pytest-postgresql`), `psycopg2-binary` (used by some proxy E2E tests), and a generated Prisma client for DB-backed proxy tests, so pytest startup matches CI without manual package installs.
Run specific test files:
```bash
uv run pytest tests/test_litellm/test_your_file.py -v
```
### Running Linting and Formatting Checks
Run all linting checks (matches CI exactly):

4
Dockerfile
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

53
Makefile
View file

@ -4,11 +4,12 @@
.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
info lint lint-dev lint-checks format \
info lint lint-inner lint-dev lint-checks format \
lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
lint-test-quality lint-test-quality-budget-update \
install-dev install-proxy-dev install-test-deps install-hooks \
install-helm-unittest check-circular-imports check-import-safety check pre-commit \
install-helm-unittest check-circular-imports check-import-safety check check-inner pre-commit \
lint-install lint-fetch-base bootstrap
# Default target
@ -35,7 +36,8 @@ help:
@echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
@echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches staging, simulates the merge)"
@echo " make lint-ruff-budget-update - Ratchet ruff-strict-budget.json limits down by what this branch fixed"
@echo " make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + basedpyright)"
@echo " make lint-test-quality - Gate the test suite against test-quality-budget.json"
@echo " make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + test quality + basedpyright)"
@echo " make check-circular-imports - Check for circular imports"
@echo " make check-import-safety - Check import safety"
@echo " make test - Run all tests"
@ -52,10 +54,17 @@ help:
@echo " make test-proxy-unit-b - Run proxy_unit_tests (p-z, ~28 files)"
@echo " make test-integration - Run integration tests"
@echo " make test-unit-helm - Run helm unit tests"
@echo ""
@echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide"
@echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine."
UV := uv
UV_RUN := $(UV) run --no-sync
# Machine-wide slot queue for the heavy targets below; python3 + stdlib only, so
# it runs before any venv exists. See scripts/gate_slot_lock.py.
GATE_SLOT_LOCK := python3 scripts/gate_slot_lock.py
LINT_DEP_INSTALL ?= install-dev
LINT_E2E_DEP_INSTALL ?= lint-install
LINT_DEP_BASE ?= lint-fetch-base
@ -73,6 +82,8 @@ info:
install-dev:
$(UV) sync --inexact --frozen
# Deliberately unqueued: provisioning is I/O bound, so it doesn't need one of the
# machine-wide slots the CPU-bound gates below share.
bootstrap:
$(UV) sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev
$(UV_RUN) python scripts/prisma_generate_if_needed.py
@ -133,11 +144,13 @@ lint-install:
$(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev
$(UV_RUN) python scripts/prisma_generate_if_needed.py
# Diff-scoped format check, identical to test-linting.yml's "Check ruff format" step:
# Diff-scoped format check, mirroring test-linting.yml's "Check ruff format" step:
# only the litellm Python files changed vs the base are checked, so a pre-existing
# format issue elsewhere doesn't block an unrelated commit.
# format issue elsewhere doesn't block an unrelated commit. Git pathspecs match
# recursively, so 'litellm/*.py' covers nested modules and the top-level files that
# CI's 'litellm/**/*.py' skips, which makes this target a superset of the CI step.
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
@files=$$(git diff --name-only origin/litellm_internal_staging...HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' || true); \
@files=$$(git diff --name-only --diff-filter=ACMR origin/litellm_internal_staging...HEAD -- 'litellm/*.py' | grep -v '^litellm/enterprise/' || true); \
if [ -z "$$files" ]; then \
echo "No changed litellm Python files to format-check."; \
else \
@ -147,6 +160,7 @@ lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
# Linting targets
lint-ruff: $(LINT_DEP_INSTALL)
cd litellm && $(UV_RUN) ruff check . && cd ..
$(UV_RUN) ruff check --config ruff-tests.toml tests
# faster linter for developing ...
# inspiration from:
@ -191,6 +205,12 @@ lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/type_discipline_gate.py --base origin/litellm_internal_staging
# Test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
# litellm module-global mutation, credential-gated skips, conftest snapshot
# inventory), counted across tests/ the same delta-vs-base way.
lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
$(UV_RUN) python scripts/test_quality_gate.py --base origin/litellm_internal_staging
# --update lowers each limit by what this branch fixed since its branch point, so
# it needs the base ref fetched to resolve the merge-base.
lint-basedpyright-budget-update: install-dev lint-fetch-base
@ -212,8 +232,11 @@ lint-ruff-budget-update: install-dev lint-fetch-base
lint-type-discipline-budget-update: install-dev lint-fetch-base
$(UV_RUN) python scripts/type_discipline_gate.py --update
# Ratchet all budgets in one shot (ruff strict + type-discipline + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-basedpyright-budget-update
lint-test-quality-budget-update: install-dev lint-fetch-base
$(UV_RUN) python scripts/test_quality_gate.py --update
# Ratchet all budgets in one shot (ruff strict + type-discipline + test quality + basedpyright)
lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-test-quality-budget-update lint-basedpyright-budget-update
check-circular-imports: $(LINT_DEP_INSTALL)
cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
@ -229,10 +252,13 @@ check-import-safety: $(LINT_DEP_INSTALL)
# does (merge-base with origin/litellm_internal_staging). Setup (env sync, Prisma client,
# base fetch) runs once up front; the checks themselves are independent, so a sub-make
# fans them out with -j and the fast ones finish under basedpyright's shadow.
lint: lint-install lint-fetch-base
lint:
@$(GATE_SLOT_LOCK) $(MAKE) lint-inner
lint-inner: lint-install lint-fetch-base
$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-test-quality lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
# Faster linting for local development (only checks changed code)
lint-dev: lint-format-changed check-circular-imports check-import-safety
@ -244,7 +270,10 @@ lint-dev: lint-format-changed check-circular-imports check-import-safety
# test-linting.yml (Python), test-litellm-ui-build.yml's frontend-lint (dashboard), and
# check-ui-api-types.yml (API-type drift), skipping any whose files aren't in scope.
# Not auto-installed as a git hook so it never slows an unrelated human commit.
check: bootstrap
check:
@$(GATE_SLOT_LOCK) $(MAKE) check-inner
check-inner: bootstrap
./scripts/pre_commit_lint.sh
pre-commit:
@ -299,7 +328,7 @@ test-unit-helm: install-helm-unittest
# LLM Translation testing targets
test-llm-translation: install-test-deps
@echo "Running LLM translation tests..."
@python .github/workflows/run_llm_translation_tests.py
@python .github/scripts/run_llm_translation_tests.py
test-llm-translation-single: install-test-deps
@echo "Running single LLM translation test file..."

1
README.md
View file

@ -292,6 +292,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |
| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | |
| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ | | | | | | | |
| [Cohere (`cohere`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ |
| [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | | | | | | | |
| [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ | | | | | | |

4
backend/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

3
backend/routes/allowlist.py
View file

@ -35,6 +35,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
# Models & routing config
"/model/",
"/v1/model/info",
"/v1/model/deprecations",
"/v2/model/",
"/model_group",
"/model_access_group/",
@ -146,11 +147,13 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
"/docs/oauth2-redirect",
"/redoc",
"/fallback/login",
"/mcp", # bare spelling of the aggregate MCP endpoint; /mcp/ prefix covers the rest
}
)
BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
{
"/swagger", # API documentation static assets belong to the backend
"/mcp", # lazily-mounted MCP sub-app serves on the backend component
}
)

42
basedpyright-code-budget.json
View file

@ -1,36 +1,36 @@
{
"reportAny": {
"limit": 23919
"limit": 17270
},
"reportArgumentType": {
"limit": 2580
"limit": 2538
},
"reportAssignmentType": {
"limit": 323
"limit": 319
},
"reportAttributeAccessIssue": {
"limit": 488
"limit": 480
},
"reportCallIssue": {
"limit": 114
"limit": 112
},
"reportConstantRedefinition": {
"limit": 40
},
"reportDeprecated": {
"limit": 213
"limit": 212
},
"reportDuplicateImport": {
"limit": 19
},
"reportExplicitAny": {
"limit": 7573
"limit": 5485
},
"reportFunctionMemberAccess": {
"limit": 7
},
"reportGeneralTypeIssues": {
"limit": 157
"limit": 101
},
"reportIncompatibleMethodOverride": {
"limit": 56
@ -45,7 +45,7 @@
"limit": 35
},
"reportInvalidTypeForm": {
"limit": 35
"limit": 34
},
"reportInvalidTypeVarUse": {
"limit": 2
@ -54,7 +54,7 @@
"limit": 0
},
"reportMissingParameterType": {
"limit": 5719
"limit": 5658
},
"reportMissingTypeArgument": {
"limit": 15656
@ -72,7 +72,7 @@
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 1069
"limit": 1055
},
"reportOptionalOperand": {
"limit": 0
@ -84,7 +84,7 @@
"limit": 56
},
"reportPrivateUsage": {
"limit": 1824
"limit": 1808
},
"reportRedeclaration": {
"limit": 8
@ -93,7 +93,7 @@
"limit": 213
},
"reportTypedDictNotRequiredAccess": {
"limit": 26
"limit": 25
},
"reportUndefinedVariable": {
"limit": 0
@ -102,7 +102,7 @@
"limit": 44831
},
"reportUnknownLambdaType": {
"limit": 113
"limit": 109
},
"reportUnknownMemberType": {
"limit": 39267
@ -114,16 +114,16 @@
"limit": 30922
},
"reportUnnecessaryCast": {
"limit": 118
"limit": 117
},
"reportUnnecessaryComparison": {
"limit": 699
"limit": 697
},
"reportUnnecessaryContains": {
"limit": 5
},
"reportUnnecessaryIsInstance": {
"limit": 853
"limit": 829
},
"reportUntypedBaseClass": {
"limit": 0
@ -132,15 +132,15 @@
"limit": 27
},
"reportUnusedClass": {
"limit": 23
"limit": 21
},
"reportUnusedFunction": {
"limit": 139
"limit": 138
},
"reportUnusedImport": {
"limit": 545
"limit": 543
},
"reportUnusedVariable": {
"limit": 146
"limit": 139
}
}

29
ci_cd/generate_model_prices_schema.py
View file

@ -27,6 +27,7 @@ EXTRA_BOOLEAN_KEYS = frozenset(
"uses_embed_content",
"use_openai_responses_path",
"bedrock_converse_supports_strict_tools",
"thinking_always_on",
}
)
@ -41,6 +42,11 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
},
"additionalProperties": False,
},
"guardrail_cost_per_unit": {
"type": "object",
"description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",
"additionalProperties": NONNEG_NUMBER,
},
"metadata": {
"type": "object",
"description": "Free-form notes about the entry (e.g. pricing derivation).",
@ -67,6 +73,11 @@ ARRAY_KEYS: dict[str, JsonSchema] = {
"description": "Output modalities the model can produce.",
"items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
},
"reasoning_effort_levels": {
"type": "array",
"description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
"items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
},
"supported_regions": {
"type": "array",
"description": "Cloud regions the model is available in ('global' or region ids).",
@ -96,6 +107,7 @@ ARRAY_KEYS: dict[str, JsonSchema] = {
"output_cost_per_token": NONNEG_NUMBER,
"output_cost_per_reasoning_token": NONNEG_NUMBER,
"cache_read_input_token_cost": NONNEG_NUMBER,
"cache_creation_input_token_cost": NONNEG_NUMBER,
"input_cost_per_query": NONNEG_NUMBER,
},
"additionalProperties": False,
@ -139,12 +151,20 @@ NUMBER_KEYS: dict[str, JsonSchema] = {
"minimum": 1,
"description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
},
"regional_endpoint_uplift_multiplier": {
"type": "number",
"minimum": 1,
"description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).",
},
}
COST_DESCRIPTIONS: dict[str, str] = {
"input_cost_per_token": "USD per prompt token.",
"output_cost_per_token": "USD per generated token.",
"output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
"google_maps_grounding_cost_per_query": (
"USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
),
"cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
"cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
"input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",
@ -200,6 +220,15 @@ def string_key_schemas(modes: tuple) -> dict[str, JsonSchema]:
"description": "Highest reasoning effort the Bedrock output_config accepts for this model.",
"enum": ["low", "medium", "high", "max", "xhigh"],
},
"default_reasoning_effort": {
"type": "string",
"description": (
"Reasoning effort the provider applies when the request omits reasoning_effort. "
"Gates whether a non-default temperature or the top_p/logprobs sampling params are "
"accepted, which hold only when the effort resolves to 'none'."
),
"enum": ["none", "minimal", "low", "medium", "high", "xhigh"],
},
"comment": STRING,
"audio_transcription_config": STRING,
}

2
codecov.yaml
View file

@ -25,6 +25,8 @@ flag_management:
carryforward: false
- name: proxy-db-schema-migration
carryforward: false
- name: circleci
carryforward: false
component_management:
individual_components:

2
cookbook/litellm_proxy_server/cli_token_usage.py
View file

@ -60,4 +60,4 @@ if __name__ == "__main__":
print("\n💡 Tips:")
print("1. Run 'litellm-proxy login' to authenticate first")
print("2. Replace 'https://your-proxy.com' with your actual proxy URL")
print("3. The token is stored locally at ~/.litellm/token.json")
print("3. The token is stored in your OS keychain, or in ~/.litellm/token.json when there is none")

5
db_scripts/partition_spend_logs.sql
View file

@ -10,6 +10,11 @@
-- partitioned, so existing installs are unaffected until you run this.
--
-- IMPORTANT
-- * After partitioning, `prisma db push` (including the proxy's
-- --use_prisma_db_push startup mode) is NOT supported: it tries to rewrite
-- the primary key back to ("request_id"), which Postgres rejects on a
-- partitioned table. The proxy detects this and exits with guidance.
-- Use the default startup path (`prisma migrate deploy`) instead.
-- * Test on a staging copy first and take a backup.
-- * Postgres cannot convert a populated table to partitioned in place, so this
-- renames the old table aside and creates a fresh partitioned table.

4
docker/Dockerfile.database
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

4
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG PROXY_EXTRAS_SOURCE=published
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.

1
enterprise/enterprise_hooks/banned_keywords.py
View file

@ -21,6 +21,7 @@ from fastapi import HTTPException
class _ENTERPRISE_BannedKeywords(CustomLogger):
enforces_request_content: bool = True
# Class variables or attributes
def __init__(self):
banned_keywords_list = litellm.banned_keywords_list

1
enterprise/enterprise_hooks/blocked_user_list.py
View file

@ -18,6 +18,7 @@ from fastapi import HTTPException
class _ENTERPRISE_BlockedUserList(CustomLogger):
enforces_request_content: bool = True
# Class variables or attributes
def __init__(self, prisma_client: Optional[PrismaClient]):
self.prisma_client = prisma_client

81
enterprise/litellm_enterprise/proxy/audit_logging_endpoints.py
View file

@ -7,7 +7,7 @@ GET - /audit/{id} - Get audit log by id
GET - /audit - Get all audit logs
"""
from typing import Any, Dict, List, Optional
from typing import TYPE_CHECKING, Final, Optional
#### AUDIT LOGGING ####
from fastapi import APIRouter, Depends, HTTPException, Query
@ -18,11 +18,16 @@ from litellm_enterprise.types.proxy.audit_logging_endpoints import (
from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import AuditLogRepository
if TYPE_CHECKING:
from prisma import models as prisma_models
router = APIRouter()
def _build_json_field_or_condition(json_key: str, value: str) -> Dict[str, Any]:
def _build_json_field_or_condition(json_key: str, value: str) -> dict[str, object]:
"""
Build an OR condition that matches a value inside a JSON column at the
given key, checking both before_value and updated_values.
@ -101,46 +106,37 @@ async def get_audit_logs(
detail={"message": CommonProxyErrors.db_not_connected_error.value},
)
# Build filter conditions
where_conditions: Dict[str, Any] = {}
if changed_by:
where_conditions["changed_by"] = changed_by
if changed_by_api_key:
where_conditions["changed_by_api_key"] = changed_by_api_key
if action:
where_conditions["action"] = action
if table_name:
where_conditions["table_name"] = table_name
if object_id:
where_conditions["object_id"] = object_id
if start_date or end_date:
date_filter: Dict[str, Any] = {}
if start_date:
date_filter["gte"] = start_date
if end_date:
date_filter["lte"] = end_date
where_conditions["updated_at"] = date_filter
date_filter: Final[dict[str, str]] = {
**({"gte": start_date} if start_date else {}),
**({"lte": end_date} if end_date else {}),
}
# JSON field filters (PostgreSQL only) — each filter is AND'd with the
# others, but checks both before_value and updated_values internally (OR).
if object_team_id:
where_conditions["AND"] = where_conditions.get("AND", []) + [
_build_json_field_or_condition("team_id", object_team_id)
]
if object_key_hash:
where_conditions["AND"] = where_conditions.get("AND", []) + [
_build_json_field_or_condition("token", object_key_hash)
]
json_field_conditions: Final[list[dict[str, object]]] = [
*([_build_json_field_or_condition("team_id", object_team_id)] if object_team_id else []),
*([_build_json_field_or_condition("token", object_key_hash)] if object_key_hash else []),
]
# Build sort conditions
order_by: Dict[str, Any] = {}
if sort_by and isinstance(sort_by, str):
order_by[sort_by] = sort_order
else:
order_by["updated_at"] = sort_order # Default sort by updated_at
# Build filter conditions
where_conditions: Final[dict[str, object]] = {
**({"changed_by": changed_by} if changed_by else {}),
**({"changed_by_api_key": changed_by_api_key} if changed_by_api_key else {}),
**({"action": action} if action else {}),
**({"table_name": table_name} if table_name else {}),
**({"object_id": object_id} if object_id else {}),
**({"updated_at": date_filter} if start_date or end_date else {}),
**({"AND": json_field_conditions} if json_field_conditions else {}),
}
order_by: Final[dict[str, str]] = (
{sort_by: sort_order} if sort_by and isinstance(sort_by, str) else {"updated_at": sort_order}
)
audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table
# Get paginated results
audit_logs = await prisma_client.db.litellm_auditlog.find_many(
audit_logs: Final = await audit_log_table.find_many(
where=where_conditions,
order=order_by,
skip=(page - 1) * page_size,
@ -148,13 +144,14 @@ async def get_audit_logs(
)
# Get total count for pagination
total_count = await prisma_client.db.litellm_auditlog.count(where=where_conditions)
total_pages = -(-total_count // page_size) # Ceiling division
total_count: Final = await audit_log_table.count(where=where_conditions)
total_pages: Final = -(-total_count // page_size) # Ceiling division
# Return paginated response
return PaginatedAuditLogResponse(
audit_logs=[
AuditLogResponse(**audit_log.model_dump()) for audit_log in audit_logs
AuditLogResponse.model_validate(audit_log.model_dump())
for audit_log in audit_logs
]
if audit_logs
else [],
@ -198,8 +195,10 @@ async def get_audit_log_by_id(
detail={"message": CommonProxyErrors.db_not_connected_error.value},
)
audit_log_table: Final[TableActions["prisma_models.LiteLLM_AuditLog"]] = AuditLogRepository(prisma_client).table
# Get the audit log by ID
audit_log = await prisma_client.db.litellm_auditlog.find_unique(where={"id": id})
audit_log: Final = await audit_log_table.find_unique(where={"id": id})
if audit_log is None:
raise HTTPException(
@ -207,4 +206,4 @@ async def get_audit_log_by_id(
)
# Convert to response model
return AuditLogResponse(**audit_log.model_dump())
return AuditLogResponse.model_validate(audit_log.model_dump())

367
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -2,8 +2,10 @@
Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if the cost has been tracked.
"""
from dataclasses import replace as dataclasses_replace
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Dict, Final, List, Literal, Optional, Tuple, cast
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
@ -13,6 +15,8 @@ from litellm.constants import (
)
if TYPE_CHECKING:
from prisma import models as prisma_models
from litellm.integrations.prometheus import PrometheusLogger
from litellm.proxy._types import LiteLLM_ManagedObjectTable
from litellm.proxy.utils import PrismaClient, ProxyLogging
@ -23,6 +27,19 @@ if TYPE_CHECKING:
CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
"completed",
"complete",
"failed",
"expired",
"cancelled",
)
TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = (
*PROVIDER_TERMINAL_BATCH_STATUSES,
"stale_expired",
)
class CheckBatchCost:
def __init__(
@ -42,6 +59,33 @@ class CheckBatchCost:
# Cached after the first poll cycle. Once we know the column is absent we skip
# the guaranteed-failing primary query on every subsequent cycle.
self._has_batch_processed_column: bool = True
self.batch_processed_support_confirmed: bool = False
@staticmethod
def _is_missing_batch_processed_column_error(err: Exception) -> bool:
message: Final = str(err).lower()
return "batch_processed" in message or "unknown column" in message or "does not exist" in message
async def confirm_batch_processed_support(self) -> None:
"""
Probe the batch_processed column before the proxy serves traffic, so the retrieve
path never sees an unconfirmed poller on a schema that has the column and accounts
inline for a batch the first poll cycle then accounts again.
"""
try:
await self.prisma_client.db.litellm_managedobjecttable.find_first(
where={"file_purpose": "batch", "batch_processed": False}
)
except Exception as probe_err:
if not self._is_missing_batch_processed_column_error(probe_err):
verbose_proxy_logger.debug(
f"CheckBatchCost: batch_processed probe failed, the poll cycle will confirm support: {probe_err}"
)
return
self._has_batch_processed_column = False
verbose_proxy_logger.warning("CheckBatchCost: batch_processed column not found, querying without it")
return
self.batch_processed_support_confirmed = True
async def _get_user_info(self, batch_id: str, user_id: Optional[str]) -> Dict[str, Any]:
"""
@ -132,11 +176,11 @@ class CheckBatchCost:
in non-terminal states as 'stale_expired'. These will never complete and
should not be polled.
"""
cutoff = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
result = await self.prisma_client.db.litellm_managedobjecttable.update_many(
cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
result: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
where={
"file_purpose": "batch",
"status": {"not_in": ["completed", "complete", "failed", "expired", "cancelled", "stale_expired"]},
"status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
"created_at": {"lt": cutoff},
},
data={"status": "stale_expired"},
@ -147,6 +191,26 @@ class CheckBatchCost:
f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
)
if not self._has_batch_processed_column:
return
# A row already in a terminal status is never rewritten by the sweep above, so
# without this it keeps a poll-page slot forever and starves newer batches.
retired: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
where={
"file_purpose": "batch",
"batch_processed": False,
"status": {"in": ["complete", "completed"]},
"created_at": {"lt": cutoff},
},
data={"batch_processed": True},
)
if retired > 0:
verbose_proxy_logger.warning(
f"CheckBatchCost: gave up on {retired} completed managed objects older than "
f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed"
)
async def _fallback_find_jobs(self) -> list:
"""Query batch jobs without the batch_processed filter (for older schemas)."""
return await self.prisma_client.db.litellm_managedobjecttable.find_many(
@ -167,6 +231,165 @@ class CheckBatchCost:
order={"created_at": "asc"},
)
async def _retire_job(self, job: "LiteLLM_ManagedObjectTable", reason: str) -> None:
"""
Take a row that can never be costed out of the poll page. Leaving it selectable
would burn one of the MAX_OBJECTS_PER_POLL_CYCLE slots on every future cycle, and
once enough such rows accumulate no newer batch is ever reached. Older schemas
without batch_processed can only be excluded through the status filter.
"""
data: Final = (
{"batch_processed": True}
if self._has_batch_processed_column
else {"status": "stale_expired"}
)
try:
await self.prisma_client.db.litellm_managedobjecttable.update(
where={"id": job.id},
data=data,
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}"
)
return
verbose_proxy_logger.warning(
f"CheckBatchCost: job {job.id} can never be costed ({reason}), "
"so it will no longer be polled"
)
async def _claim_job_for_costing(self, job: "LiteLLM_ManagedObjectTable") -> bool:
"""
Atomically flip batch_processed from false to true, returning whether this pod won
the row. Every pod and uvicorn worker schedules its own poller against the shared
table, so without this compare-and-swap two of them can select the same completed
batch in one window and both emit an aretrieve_batch spend log for it. Schemas
without the column can't be claimed, so they keep the pre-existing behavior.
Called immediately before the spend log is written rather than before the results
fetch, because batch_processed is also what holds off deletion of the files that
fetch reads and what keeps an unbilled row selectable by the next poll cycle.
"""
if not self._has_batch_processed_column:
return True
try:
claimed: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
where={"id": job.id, "batch_processed": False},
data={"batch_processed": True},
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to claim job {job.id} for cost tracking: {db_err}"
)
return False
return claimed > 0
async def _release_job_claim(self, job: "LiteLLM_ManagedObjectTable") -> None:
"""Give a claimed row back once billing it failed, so a later poll cycle retries it.
Safe to match on batch_processed=True: while this poller is active the retrieve
path leaves the column alone (batch_cost_poller_is_active), so a true value here
is always this pod's own claim.
"""
if not self._has_batch_processed_column:
return
try:
await self.prisma_client.db.litellm_managedobjecttable.update_many(
where={"id": job.id, "batch_processed": True},
data={"batch_processed": False},
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to release the claim on job {job.id}, "
f"so its cost will not be retried: {db_err}"
)
@staticmethod
def _has_unified_id_without_model(job: "LiteLLM_ManagedObjectTable") -> bool:
"""A unified id that decodes but carries no model_id can never be routed."""
from litellm.proxy.openai_files_endpoints.common_utils import (
convert_b64_uid_to_unified_uid,
get_model_id_from_unified_batch_id,
)
decoded: Final = convert_b64_uid_to_unified_uid(job.unified_object_id)
return (
decoded != job.unified_object_id
and get_model_id_from_unified_batch_id(decoded) is None
)
@staticmethod
def _is_batch_gone_at_provider(error: Exception, batch_id: str) -> bool:
"""
A 404 naming the batch means the provider dropped its record of it, so no later
retrieve can ever succeed. A 404 about anything else, a renamed Azure deployment
or a fallback deployment that never saw this batch, is still fixable in config, so
it keeps retrying.
"""
import openai
from litellm.exceptions import NotFoundError
return isinstance(error, (NotFoundError, openai.NotFoundError)) and batch_id in str(error)
def _batch_deployment_exists(self, model_id: str) -> bool:
"""A 404 only proves the batch is gone when it came from the batch's own
deployment. Once that deployment leaves the router, default fallbacks can
silently send the retrieve to a provider that never saw the batch, so its
404 must not retire the row; the staleness sweep bounds it instead."""
return self.llm_router.get_deployment(model_id=model_id) is not None
@staticmethod
def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
"""A 404 naming the output file means there is nothing to fetch on this or any
later poll: providers like Vertex AI advertise an output path for every batch,
including terminal ones that never wrote it. Any other failure may be
transient, so it keeps retrying until the staleness sweep bounds it."""
import openai
from litellm.exceptions import NotFoundError
if not output_file_id:
return False
return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
async def _finalize_unbilled_terminal_job(
self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
) -> None:
"""Persist a terminal batch that has nothing billable, converting any raw
provider file ids to managed ids, and take it out of the poll page."""
try:
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
ensure_batch_response_managed_file_ids,
)
response.id = job.unified_object_id
await ensure_batch_response_managed_file_ids(
response=response,
managed_files_obj=self.proxy_logging_obj.get_proxy_hook("managed_files"),
prisma_client=self.prisma_client,
verbose_proxy_logger=verbose_proxy_logger,
db_batch_object=job,
unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
)
update_data: Final[dict] = {
"status": response.status,
"file_object": response.model_dump_json(),
**({"batch_processed": True} if self._has_batch_processed_column else {}),
}
await self.prisma_client.db.litellm_managedobjecttable.update(
where={"id": job.id},
data=update_data,
)
verbose_proxy_logger.info(
f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}"
)
@staticmethod
def _record_error(
prom_logger: Optional["PrometheusLogger"], error_type: str
@ -398,17 +621,20 @@ class CheckBatchCost:
"""
Fetch a completed batch's results, compute cost/usage, and emit the
aretrieve_batch spend log. Returns (model_name, llm_provider) on
success, None when the job can't be routed to a deployment. Raises on
results-fetch or cost-computation failures so the caller can leave the
job unprocessed and retry it on a later poll.
success, None when the job can't be routed to a deployment or when
another pod claimed it. Raises on results-fetch or cost-computation
failures so the caller can leave the job unprocessed and retry it on a
later poll.
"""
from litellm.batches.batch_utils import (
count_error_file_failed_requests,
_get_file_content_as_dictionary,
calculate_batch_cost_and_usage,
)
from litellm.files.main import afile_content
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
)
@ -446,6 +672,7 @@ class CheckBatchCost:
credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
_file_content = await afile_content(
file_id=raw_output_file_id,
_litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
**credentials,
)
@ -528,19 +755,41 @@ class CheckBatchCost:
f"{_file_attr}={_raw_file_id!r}: {_e}"
)
# Pass deployment model_info so custom batch pricing
# (input_cost_per_token_batches etc.) is used for cost calc
deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}
batch_cost, batch_usage, batch_models = (
await calculate_batch_cost_and_usage(
file_content_dictionary=file_content_as_dict,
custom_llm_provider=llm_provider, # type: ignore
model_name=model_name,
model_info=deployment_model_info, # type: ignore[arg-type]
# Pass the deployment's router-registered pricing (litellm_params custom
# rates merged with the model's published rates) so custom batch pricing
# (input_cost_per_token_batches etc.) is used for cost calc, exactly as
# the inline retrieve path does.
deployment_model_info = deployment_pricing_model_info(
model_id=model_id,
deployment_model=litellm_model_name,
)
batch_file_provider: Final = cast(
Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"], llm_provider
)
output_file_result: Final = await calculate_batch_cost_and_usage(
file_content_dictionary=file_content_as_dict,
custom_llm_provider=batch_file_provider,
model_name=model_name,
model_info=deployment_model_info,
)
error_file_failed_requests: Final = await count_error_file_failed_requests(
response,
custom_llm_provider=batch_file_provider,
litellm_params={
**credentials,
"_litellm_internal_model_credentials": MappingProxyType(dict(credentials)),
},
)
batch_result: Final = (
output_file_result
if not error_file_failed_requests
else dataclasses_replace(
output_file_result,
failed_requests=output_file_result.failed_requests + error_file_failed_requests,
)
)
logging_obj = LiteLLMLogging(
model=batch_models[0],
model=batch_result.models[0],
messages=[{"role": "user", "content": "<retrieve_batch>"}],
stream=False,
call_type="aretrieve_batch",
@ -562,12 +811,25 @@ class CheckBatchCost:
optional_params={},
)
await logging_obj.async_success_handler(
result=response,
batch_cost=batch_cost,
batch_usage=batch_usage,
batch_models=batch_models,
)
if not await self._claim_job_for_costing(job):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} (job {job.id}) was claimed by another pod "
"in this window, so its cost is already being tracked there"
)
return None
try:
await logging_obj.async_success_handler(
result=response,
batch_cost=batch_result.cost,
batch_usage=batch_result.usage,
batch_models=batch_result.models,
batch_successful_requests=batch_result.successful_requests,
batch_failed_requests=batch_result.failed_requests,
)
except Exception:
await self._release_job_claim(job)
raise
# Record batch duration (completed_at - created_at)
if prom_logger and response.completed_at and response.created_at:
@ -631,8 +893,9 @@ class CheckBatchCost:
take=MAX_OBJECTS_PER_POLL_CYCLE,
order={"created_at": "asc"},
)
self.batch_processed_support_confirmed = True
except Exception as query_err:
if "batch_processed" not in str(query_err).lower() and "unknown column" not in str(query_err).lower() and "does not exist" not in str(query_err).lower():
if not self._is_missing_batch_processed_column_error(query_err):
raise
# Permanent schema gap — cache the result so future cycles skip straight to fallback
self._has_batch_processed_column = False
@ -645,6 +908,8 @@ class CheckBatchCost:
for job in jobs:
routing = self._resolve_job_routing(job, prom_logger)
if routing is None:
if self._has_unified_id_without_model(job):
await self._retire_job(job, "unified object id has no model id")
continue
model_id, batch_id = routing
@ -667,11 +932,13 @@ class CheckBatchCost:
)
if prom_logger:
prom_logger.record_check_batch_cost_error("provider_retrieval_error")
if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id):
await self._retire_job(job, f"batch {batch_id} no longer exists at the provider")
continue
## RETRIEVE THE BATCH JOB OUTPUT FILE
if (
response.status == "completed"
response.status in PROVIDER_TERMINAL_BATCH_STATUSES
and response.output_file_id is not None
):
try:
@ -683,6 +950,15 @@ class CheckBatchCost:
prom_logger=prom_logger,
)
except Exception as tracking_err:
if self._is_output_file_gone_at_provider(
tracking_err, response.output_file_id
) and self._batch_deployment_exists(model_id):
verbose_proxy_logger.warning(
f"CheckBatchCost: output file {response.output_file_id} of batch {batch_id} "
f"does not exist at the provider; retiring job {job.id} unbilled"
)
await self._finalize_unbilled_terminal_job(job, response)
continue
verbose_proxy_logger.error(
f"CheckBatchCost: failed to track cost for batch {batch_id} "
f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
@ -698,7 +974,7 @@ class CheckBatchCost:
# mark the job as complete
try:
update_data: dict = {
"status": "complete",
"status": response.status if response.status != "completed" else "complete",
"file_object": response.model_dump_json(),
}
if self._has_batch_processed_column:
@ -712,39 +988,18 @@ class CheckBatchCost:
f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
)
elif response.status in ("failed", "expired", "cancelled"):
try:
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
ensure_batch_response_managed_file_ids,
)
elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
from litellm.proxy.openai_files_endpoints.common_utils import (
_completed_batch_safe_to_retire,
)
response.id = job.unified_object_id
await ensure_batch_response_managed_file_ids(
response=response,
managed_files_obj=self.proxy_logging_obj.get_proxy_hook("managed_files"),
prisma_client=self.prisma_client,
verbose_proxy_logger=verbose_proxy_logger,
db_batch_object=job,
unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
)
update_data = {
"status": response.status,
"file_object": response.model_dump_json(),
}
if self._has_batch_processed_column:
update_data["batch_processed"] = True
await self.prisma_client.db.litellm_managedobjecttable.update(
where={"id": job.id},
data=update_data,
)
if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
verbose_proxy_logger.info(
f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
)
except Exception as db_err:
verbose_proxy_logger.error(
f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}"
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
)
continue
await self._finalize_unbilled_terminal_job(job, response)
# Record polling run metrics (always, even if nothing was processed)
if prom_logger:

10
enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py
View file

@ -1,6 +1,8 @@
"""
Polls LiteLLM_ManagedObjectTable to check if the response is complete.
Cost tracking is handled automatically by the get-responses call.
Cost tracking is handled by the get-responses call, which prices normally only because the
poll stamps itself with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN; user-facing reads of the
same route are non-inference and free.
"""
from datetime import datetime, timedelta, timezone
@ -9,12 +11,14 @@ from typing import TYPE_CHECKING, Dict, Optional, cast
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
INTERNAL_CALL_ORIGIN_METADATA_KEY,
MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
MAX_OBJECTS_PER_POLL_CYCLE,
STALE_OBJECT_CLEANUP_BATCH_SIZE,
)
from litellm.responses.utils import ResponsesAPIRequestUtils
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient, ProxyLogging
@ -113,7 +117,8 @@ class CheckResponsesCost:
Check if background responses are complete and track their cost.
- Get all status="queued" or "in_progress" and file_purpose="response" jobs
- Query the provider to check if response is complete
- Cost is automatically tracked by the get-responses call
- Cost is tracked by the get-responses call, billed because the poll is stamped
with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
- Mark responses in a terminal state as complete in the database
"""
try:
@ -153,6 +158,7 @@ class CheckResponsesCost:
# Prepare metadata with model information for cost tracking
litellm_metadata = {
"user_api_key_user_id": job.created_by or "default-user-id",
INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
}
# Add model information if available

190
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -36,14 +36,18 @@ from litellm.llms.base_llm.managed_resources.isolation import (
build_list_page,
build_owner_filter,
can_access_resource,
resolve_resource_owner_id,
)
from litellm.proxy._types import (
CallTypes,
LiteLLM_ManagedFileTable,
LiteLLM_ManagedObjectTable,
ProxyException,
UserAPIKeyAuth,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
FILE_LIST_CONTINUATION_CHUNK_SIZE,
MAX_FILE_LIST_LIMIT,
_is_base64_encoded_unified_file_id,
apply_unified_file_ids,
ensure_batch_response_managed_file_ids,
@ -53,15 +57,20 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
map_raw_file_ids_to_unified,
normalize_mime_type_for_provider,
resolve_managed_output_file_model_name,
validate_file_list_limit,
validate_file_list_purpose,
)
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
request_tags_from_metadata,
)
from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccessIssue]
AllMessageValues,
AsyncCursorPage,
ChatCompletionFileObject,
CreateFileRequest,
FileListPage,
FileObject,
OpenAIFileObject,
OpenAIFilesPurpose,
ResponsesAPIResponse,
)
from litellm.types.utils import (
@ -140,7 +149,14 @@ class _ManagedFileRow(Protocol):
class _ManagedFileTableActions(Protocol):
async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ...
async def find_many(self, where: Mapping[str, object]) -> Sequence[_ManagedFileRow]: ...
async def find_many(
self,
where: Mapping[str, object],
take: int = ...,
order: Union[Mapping[str, str], Sequence[Mapping[str, str]]] = ...,
cursor: Mapping[str, str] = ...,
skip: int = ...,
) -> Sequence[_ManagedFileRow]: ...
async def upsert(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, object]]) -> _ManagedFileRow: ...
@ -207,7 +223,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
file_object=file_object,
model_mappings=model_mappings,
flat_model_file_ids=list(model_mappings.values()),
created_by=user_api_key_dict.user_id,
created_by=resolve_resource_owner_id(user_api_key_dict),
team_id=user_api_key_dict.team_id,
updated_by=user_api_key_dict.user_id,
)
@ -223,7 +239,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"unified_file_id": file_id,
"model_mappings": json.dumps(model_mappings),
"flat_model_file_ids": list(model_mappings.values()),
"created_by": user_api_key_dict.user_id,
"created_by": resolve_resource_owner_id(user_api_key_dict),
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
}
@ -327,7 +343,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
"file_object": file_object.model_dump_json(),
"model_object_id": model_object_id,
"file_purpose": file_purpose,
"created_by": user_api_key_dict.user_id,
"created_by": resolve_resource_owner_id(user_api_key_dict),
"team_id": user_api_key_dict.team_id,
"updated_by": user_api_key_dict.user_id,
"status": file_object.status,
@ -420,13 +436,26 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
# This is because the encoded object ids stored in the managed objects table do not contain the provider information
# To support provider filtering, we would need to store the provider information in the encoded object ids
if provider:
raise Exception("Filtering by 'provider' is not supported when using managed batches.")
raise ProxyException(
message="Filtering by 'provider' is not supported when using managed batches.",
type="invalid_request_error",
param="provider",
code=400,
)
# Model name filtering is not supported for managed batches
# This is because the encoded object ids stored in the managed objects table do not contain the model name
# A hash of the model name + litellm_params for the model name is encoded as the model id. This is not sufficient to reliably map the target model names to the model ids.
if target_model_names:
raise Exception("Filtering by 'target_model_names' is not supported when using managed batches.")
raise ProxyException(
message="Filtering by 'target_model_names' is not supported when using managed batches.",
type="invalid_request_error",
param="target_model_names",
code=400,
)
if limit == 0:
return build_list_page([])
owner_filter = build_owner_filter(user_api_key_dict)
if owner_filter is None:
@ -445,19 +474,56 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
)
page_size: Final = min(limit or 20, 100)
cursor_args: _CursorPageArgs = {"cursor": {"unified_object_id": after}, "skip": 1} if after else {}
batches = await _managed_object_table(self.prisma_client).find_many(
where=where_clause,
take=page_size + 1,
order=[{"created_at": "desc"}, {"unified_object_id": "desc"}],
**cursor_args,
matches: Final = await self._collect_listed_batches(
where_clause=where_clause,
after=after,
wanted=page_size + 1,
user_api_key_dict=user_api_key_dict,
)
return build_list_page(list(matches[:page_size]), has_more=len(matches) > page_size)
has_more = len(batches) > page_size
async def _collect_listed_batches(
self,
where_clause: Mapping[str, object],
after: Optional[str],
wanted: int,
user_api_key_dict: UserAPIKeyAuth,
) -> tuple[LiteLLMBatch, ...]:
"""Read chunks newest-first until ``wanted`` batches survive parsing and
file-id resolution or the caller's rows run out, so a run of rows that will
not parse refills the page instead of emptying it. The first chunk is
``wanted`` rows, so a healthy page still costs one query; a scan that has to
continue widens to ``FILE_LIST_CONTINUATION_CHUNK_SIZE`` like ``afile_list``,
and every chunk advances the keyset cursor, so the walk ends once the
caller's rows are exhausted."""
matches: tuple[LiteLLMBatch, ...] = () # rebind-ok: accumulates survivors across chunks
cursor_id: Optional[str] = after # rebind-ok: keyset cursor advances to each chunk's last row
chunk_size: int = wanted # rebind-ok: widens once a scan has to continue past the first chunk
while len(matches) < wanted:
cursor_args: _CursorPageArgs = {"cursor": {"unified_object_id": cursor_id}, "skip": 1} if cursor_id else {}
chunk = await _managed_object_table(self.prisma_client).find_many(
where=where_clause,
take=chunk_size,
order=[{"created_at": "desc"}, {"unified_object_id": "desc"}],
**cursor_args,
)
matches = matches + await self._resolve_listed_rows(
rows=chunk, wanted=wanted - len(matches), user_api_key_dict=user_api_key_dict
)
if len(chunk) < chunk_size:
break
cursor_id = chunk[-1].unified_object_id
chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
return matches
async def _resolve_listed_rows(
self,
rows: "Sequence[PrismaManagedObjectRow]",
wanted: int,
user_api_key_dict: UserAPIKeyAuth,
) -> tuple[LiteLLMBatch, ...]:
parsed_rows: Final = tuple(
(row, batch_obj) for row in batches[:page_size] if (batch_obj := _parse_managed_batch_row(row)) is not None
(row, batch_obj) for row in rows if (batch_obj := _parse_managed_batch_row(row)) is not None
)
unified_id_by_raw_id: Final = await map_raw_file_ids_to_unified(
raw_file_ids=frozenset(
@ -468,19 +534,19 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
),
prisma_client=self.prisma_client,
)
resolved_batches: Final = [
await self._resolve_listed_batch(
resolved: Final[list[LiteLLMBatch]] = [] # mutable-ok: resolution stops as soon as the page is full
for row, batch_obj in parsed_rows:
if len(resolved) == wanted:
break
resolved_batch = await self._resolve_listed_batch(
row=row,
batch_obj=batch_obj,
unified_id_by_raw_id=unified_id_by_raw_id,
user_api_key_dict=user_api_key_dict,
)
for row, batch_obj in parsed_rows
]
return build_list_page(
[batch_obj for batch_obj in resolved_batches if batch_obj is not None],
has_more=has_more,
)
if resolved_batch is not None:
resolved.append(resolved_batch)
return tuple(resolved)
async def _resolve_listed_batch(
self,
@ -1146,6 +1212,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
## Check if unified_file_id is in the response
unified_file_id = response._hidden_params.get("unified_file_id") # managed file id
unified_batch_id = response._hidden_params.get("unified_batch_id") # managed batch id
is_batch_create: Final = unified_file_id is not None
model_id = cast(Optional[str], response._hidden_params.get("model_id"))
model_name = cast(Optional[str], response._hidden_params.get("model_name"))
@ -1216,6 +1283,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_mappings={model_id: provider_file_id},
user_api_key_dict=user_api_key_dict,
)
request_metadata: Final = data.get("litellm_metadata")
await self.store_unified_object_id(
unified_object_id=response.id,
file_object=response,
@ -1223,6 +1291,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
model_object_id=original_response_id,
file_purpose="batch",
user_api_key_dict=user_api_key_dict,
request_tags=request_tags_from_metadata(request_metadata if isinstance(request_metadata, dict) else {}),
persist_attribution=is_batch_create,
)
# Only record batch creation metric on actual create (not retrieve/cancel).
@ -1344,12 +1414,76 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def afile_list(
self,
purpose: Optional[OpenAIFilesPurpose],
purpose: Optional[str],
litellm_parent_otel_span: Optional[Span],
user_api_key_dict: UserAPIKeyAuth,
limit: Optional[int] = None,
after: Optional[str] = None,
**data: Dict,
) -> List[OpenAIFileObject]:
"""Handled in files_endpoints.py"""
return []
) -> FileListPage:
"""List the managed files the caller owns, newest first.
Pagination is keyset based on ``unified_file_id`` so a key that owns
every file on the proxy still reads one bounded page at a time.
``purpose`` is applied after parsing, because the managed file table
keeps it inside the ``file_object`` blob instead of a column, and rows
whose blob will not parse drop out there too, so a chunk of rows can
yield fewer matches than the page holds. Successive chunks are read
until the page is full or the caller's rows run out, which keeps
``data`` non-empty while matches remain and its last id usable as the
next cursor. A first chunk that fills the page costs one query; once a
scan has to continue past it, the chunk widens to
``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
that many rows instead of one per page. That bound is per query, not
per request: the work is still linear in the rows the caller owns, and
a filter matching nothing reads every one of them, with no index
covering either the owner filter or the sort.
"""
validate_file_list_limit(limit)
validate_file_list_purpose(purpose)
owner_filter: Final = build_owner_filter(user_api_key_dict)
if owner_filter is None:
return FileListPage(**build_list_page([]))
if after:
cursor_row = await _managed_file_table(self.prisma_client).find_first(
where={**owner_filter, "unified_file_id": after}
)
if cursor_row is None:
raise ProxyException(
message=f"Invalid 'after' cursor: no file found with id '{after}'.",
type="invalid_request_error",
param="after",
code=400,
openai_code="invalid_value",
)
page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
matches: Final[List[OpenAIFileObject]] = []
cursor_id = after
chunk_size = page_size + 1
while len(matches) <= page_size:
cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
chunk = await _managed_file_table(self.prisma_client).find_many(
where=owner_filter,
take=chunk_size,
order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
**cursor_args,
)
matches.extend(
parsed_file_object.model_copy(update={"id": row.unified_file_id})
for row in chunk
if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
and (purpose is None or parsed_file_object.purpose == purpose)
)
if len(chunk) < chunk_size:
break
cursor_id = chunk[-1].unified_file_id
chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
def _is_batch_polling_enabled(self) -> bool:
"""

259
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -11,10 +11,11 @@ Endpoints for /project operations
#### PROJECT MANAGEMENT ####
import json
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING
from collections.abc import Sequence
from typing import TYPE_CHECKING, Final
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
@ -26,17 +27,55 @@ from litellm.proxy.management_helpers.utils import (
management_endpoint_wrapper,
)
from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
if TYPE_CHECKING:
from prisma import models as prisma_models
from prisma.actions import LiteLLM_TeamTableActions
from litellm import Router
router = APIRouter()
_OBJECT_PERMISSION_PAYLOAD: Final = TypeAdapter(dict[str, object])
def _team_table(prisma_client: PrismaClient) -> "LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable]":
team_table: LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable] = prisma_client.db.litellm_teamtable
return team_table
def _team_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_TeamTable"]:
return TeamRepository(prisma_client).table
def _project_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_ProjectTable"]:
return ProjectRepository(prisma_client).table
def _verification_token_table(
prisma_client: PrismaClient,
) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
return VerificationTokenRepository(prisma_client).table
def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
return BudgetRepository(prisma_client).table
def _object_permission_table(
prisma_client: PrismaClient,
) -> TableActions["prisma_models.LiteLLM_ObjectPermissionTable"]:
return ObjectPermissionRepository(prisma_client).table
def _user_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_UserTable"]:
return UserRepository(prisma_client).table
def _jsonified(prisma_client: PrismaClient, payload: dict[str, object]) -> dict[str, object]:
jsonified: dict[str, object] = prisma_client.jsonify_object(payload)
return jsonified
async def _check_user_permission_for_project(
@ -137,7 +176,7 @@ def _check_team_project_limits(
# --- Validate project models are a subset of team models ---
project_models = data.models
team_models = team_object.models or []
team_models: list[str] = team_object.models or []
if project_models and len(team_models) > 0:
# If team has 'all-proxy-models', skip validation as it allows all models
if SpecialModelNames.all_proxy_models.value not in team_models:
@ -180,6 +219,114 @@ def _check_team_project_limits(
)
def _project_models_missing_positive_quota(
models: list[str] | None,
rpm_limits: Mapping[str, object] | None,
tpm_limits: Mapping[str, object] | None,
) -> list[str]:
"""Return the models that lack a positive `rpm` AND `tpm` quota.
A valid quota is a positive integer; null, zero, and negative are rejected
because downstream rate limiters treat a non-positive limit as immediately
exhausted (every request blocked).
"""
def _is_positive(value: object) -> bool:
return isinstance(value, int) and not isinstance(value, bool) and value > 0
rpm = rpm_limits or {}
tpm = tpm_limits or {}
return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))]
def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
def _project_models_expanding_at_request_time(
models: Sequence[str] | None, access_group_names: frozenset[str]
) -> tuple[str, ...]:
"""Entries project auth expands to many concrete models (`all-proxy-models`, `*` patterns,
access groups). The rate limiter looks quotas up by the exact requested model name, so a
quota keyed on one of these entries is never applied."""
return tuple(
model
for model in (models or ())
if model == SpecialModelNames.all_proxy_models.value or "*" in model or model in access_group_names
)
def _raise_on_project_models_expanding_at_request_time(
models: Sequence[str] | None, access_group_names: frozenset[str]
) -> None:
expanding: Final = _project_models_expanding_at_request_time(models, access_group_names)
if not expanding:
return
raise HTTPException(
status_code=400,
detail={
"error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while 'enforce_project_model_quota' is enabled. List concrete model names instead."
},
)
def _raise_on_missing_project_model_quota(
data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset()
) -> None:
"""Require a positive `rpm`/`tpm` quota for every model on project CREATE.
`model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request
model's `set_model_info` validator, so they are read from there.
Only invoked when `general_settings.enforce_project_model_quota` is enabled
(default off), so it is opt-in and does not change behavior for existing users.
"""
_raise_on_project_models_expanding_at_request_time(data.models, access_group_names)
metadata = data.metadata or {}
missing = _project_models_missing_positive_quota(
data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit")
)
if not missing:
return
raise HTTPException(
status_code=400,
detail={
"error": f"models {missing} added to project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
},
)
def _raise_on_missing_project_model_quota_on_update(
data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset()
) -> None:
"""Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE.
`/project/update` replaces `models` and `metadata` when they are provided, so the
check runs on what the project WILL look like: a partial update that doesn't touch
models/quota keeps the existing values, while one that adds a model or clears a
model's quota must leave every resulting model with a positive limit.
Only invoked when `general_settings.enforce_project_model_quota` is enabled
(default off), so it is opt-in and does not change behavior for existing users.
"""
resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or [])
resulting_metadata = (
data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {})
)
_raise_on_project_models_expanding_at_request_time(resulting_models, access_group_names)
missing = _project_models_missing_positive_quota(
resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit")
)
if not missing:
return
raise HTTPException(
status_code=400,
detail={
"error": f"models {missing} would be left on the project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
},
)
async def _create_budget_for_project(
data: NewProjectRequest,
user_id: str | None,
@ -188,13 +335,13 @@ async def _create_budget_for_project(
) -> str:
"""Create a budget for the project and return budget_id."""
budget_params = LiteLLM_BudgetTable.model_fields.keys()
_json_data: Mapping[str, object] = data.json(exclude_none=True)
_json_data: dict[str, object] = data.model_dump(exclude_none=True)
_budget_data = {k: v for k, v in _json_data.items() if k in budget_params}
budget_row = LiteLLM_BudgetTable.model_validate(_budget_data)
new_budget = prisma_client.jsonify_object(budget_row.json(exclude_none=True))
new_budget = _jsonified(prisma_client, budget_row.model_dump(exclude_none=True))
_budget: prisma_models.LiteLLM_BudgetTable = await prisma_client.db.litellm_budgettable.create(
_budget: Final = await _budget_table(prisma_client).create(
data={
**new_budget,
"created_by": user_id or litellm_proxy_admin_name,
@ -217,17 +364,15 @@ async def _set_project_object_permission(
return None
if data.object_permission is not None:
created_object_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
await prisma_client.db.litellm_objectpermissiontable.create(
data=data.object_permission.model_dump(exclude_none=True),
)
created_object_permission: Final = await _object_permission_table(prisma_client).create(
data=data.object_permission.model_dump(exclude_none=True),
)
del data.object_permission
return created_object_permission.object_permission_id
return None
def _remove_budget_fields_from_project_data(project_data: dict) -> dict:
def _remove_budget_fields_from_project_data(project_data: dict[str, object]) -> dict[str, object]:
"""
Remove budget fields from project data.
Budget fields belong to LiteLLM_BudgetTable, not LiteLLM_ProjectTable.
@ -327,7 +472,9 @@ async def new_project(
```
"""
from litellm.proxy.proxy_server import (
general_settings,
litellm_proxy_admin_name,
llm_router,
premium_user,
prisma_client,
)
@ -374,6 +521,10 @@ async def new_project(
data=data,
)
# Opt-in (default off): require rpm/tpm for every model added to the project.
if general_settings.get("enforce_project_model_quota", False):
_raise_on_missing_project_model_quota(data, _router_access_group_names(llm_router))
# Check if user has permission to create projects for this team
# only team admins can create projects for their team
has_permission = await _check_user_permission_for_project(
@ -396,9 +547,7 @@ async def new_project(
data.project_id = str(uuid.uuid4())
else:
# Check if project_id already exists
existing_project = await prisma_client.db.litellm_projecttable.find_unique(
where={"project_id": data.project_id}
)
existing_project = await _project_table(prisma_client).find_unique(where={"project_id": data.project_id})
if existing_project is not None:
raise ProxyException(
message=f"Project id = {data.project_id} already exists. Please use a different project id.",
@ -423,11 +572,14 @@ async def new_project(
)
# Create project row (following organization_endpoints.py pattern)
project_row = LiteLLM_ProjectTable(
**data.json(exclude_none=True),
object_permission_id=object_permission_id,
created_by=user_api_key_dict.user_id or litellm_proxy_admin_name,
updated_by=user_api_key_dict.user_id or litellm_proxy_admin_name,
project_row_payload: dict[str, object] = data.model_dump(exclude_none=True)
project_row = LiteLLM_ProjectTable.model_validate(
{
**project_row_payload,
"object_permission_id": object_permission_id,
"created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
"updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
}
)
for field in LiteLLM_ManagementEndpoint_MetadataFields:
@ -438,16 +590,14 @@ async def new_project(
value=getattr(data, field),
)
new_project_row = prisma_client.jsonify_object(project_row.json(exclude_none=True))
new_project_row = _jsonified(prisma_client, project_row.model_dump(exclude_none=True))
# Remove budget fields (following organization_endpoints.py pattern)
new_project_row = _remove_budget_fields_from_project_data(new_project_row)
verbose_proxy_logger.info(f"new_project_row: {json.dumps(new_project_row, indent=2)}")
response: prisma_models.LiteLLM_ProjectTable = await prisma_client.db.litellm_projecttable.create(
data={
**new_project_row, # type: ignore
},
response: Final = await _project_table(prisma_client).create(
data={**new_project_row},
include={"litellm_budget_table": True},
)
@ -512,7 +662,9 @@ async def update_project(
```
"""
from litellm.proxy.proxy_server import (
general_settings,
litellm_proxy_admin_name,
llm_router,
premium_user,
prisma_client,
user_api_key_cache,
@ -560,7 +712,7 @@ async def update_project(
# Fetch existing project
existing_project: (
prisma_models.LiteLLM_ProjectTable | None
) = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": data.project_id})
) = await _project_table(prisma_client).find_unique(where={"project_id": data.project_id})
if existing_project is None:
raise ProxyException(
@ -616,9 +768,14 @@ async def update_project(
data=data,
)
# Opt-in (default off): require rpm/tpm for every model the update would leave on the project.
if general_settings.get("enforce_project_model_quota", False):
_raise_on_missing_project_model_quota_on_update(
data, existing_project, _router_access_group_names(llm_router)
)
# Prepare update data
update_data = data.json(exclude_none=True, exclude={"project_id"})
update_data = prisma_client.jsonify_object(update_data)
update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
# Handle budget updates
@ -627,7 +784,7 @@ async def update_project(
if budget_updates and existing_project.budget_id:
# Update existing budget
await prisma_client.db.litellm_budgettable.update(
await _budget_table(prisma_client).update(
where={"budget_id": existing_project.budget_id},
data={
**budget_updates,
@ -642,33 +799,33 @@ async def update_project(
if "object_permission" in update_data:
object_permission_data = update_data.pop("object_permission")
if object_permission_data:
object_permission_payload: Final = _OBJECT_PERMISSION_PAYLOAD.validate_python(object_permission_data)
if existing_project.object_permission_id:
# Update existing permission
await prisma_client.db.litellm_objectpermissiontable.update(
await _object_permission_table(prisma_client).update(
where={"object_permission_id": existing_project.object_permission_id},
data=object_permission_data,
data=object_permission_payload,
)
else:
# Create new permission
created_permission: prisma_models.LiteLLM_ObjectPermissionTable = (
await prisma_client.db.litellm_objectpermissiontable.create(
data=object_permission_data,
)
created_permission: Final = await _object_permission_table(prisma_client).create(
data=object_permission_payload,
)
update_data["object_permission_id"] = created_permission.object_permission_id
# Handle metadata fields
for field in LiteLLM_ManagementEndpoint_MetadataFields:
if field in update_data:
if update_data.get("metadata") is None:
update_data["metadata"] = {}
update_data["metadata"][field] = update_data.pop(field)
existing_metadata = update_data.get("metadata")
metadata_dict: dict[str, object] = existing_metadata if isinstance(existing_metadata, dict) else {}
metadata_dict[field] = update_data.pop(field)
update_data["metadata"] = metadata_dict
# Remove budget fields (following organization_endpoints.py pattern)
update_data = _remove_budget_fields_from_project_data(update_data)
# Update project
updated_project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.update(
updated_project: Final = await _project_table(prisma_client).update(
where={"project_id": data.project_id},
data=update_data,
include={"litellm_budget_table": True, "object_permission": True},
@ -748,11 +905,11 @@ async def delete_project(
detail={"error": "Only admins can delete projects"},
)
deleted_projects = []
deleted_projects: list[prisma_models.LiteLLM_ProjectTable | None] = []
for project_id in data.project_ids:
# Check if project exists
existing_project = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": project_id})
existing_project = await _project_table(prisma_client).find_unique(where={"project_id": project_id})
if existing_project is None:
raise ProxyException(
@ -765,7 +922,7 @@ async def delete_project(
# Check if there are any keys associated with this project
associated_keys: Sequence[
prisma_models.LiteLLM_VerificationToken
] = await prisma_client.db.litellm_verificationtoken.find_many(where={"project_id": project_id})
] = await _verification_token_table(prisma_client).find_many(where={"project_id": project_id})
if len(associated_keys) > 0:
raise ProxyException(
@ -778,7 +935,7 @@ async def delete_project(
# Delete the project
deleted_project: (
prisma_models.LiteLLM_ProjectTable | None
) = await prisma_client.db.litellm_projecttable.delete(where={"project_id": project_id})
) = await _project_table(prisma_client).delete(where={"project_id": project_id})
await delete_cached_project_object(
project_id=project_id,
@ -829,7 +986,7 @@ async def project_info(
)
# Fetch project
project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.find_unique(
project: prisma_models.LiteLLM_ProjectTable | None = await _project_table(prisma_client).find_unique(
where={"project_id": project_id},
include={"litellm_budget_table": True, "object_permission": True},
)
@ -901,19 +1058,19 @@ async def list_projects(
if user_api_key_has_admin_view(user_api_key_dict):
projects: Sequence[
prisma_models.LiteLLM_ProjectTable
] = await prisma_client.db.litellm_projecttable.find_many(
] = await _project_table(prisma_client).find_many(
include={"litellm_budget_table": True, "object_permission": True}
)
else:
# Look up the user's team memberships via the reverse-index on
# LiteLLM_UserTable.teams (maintained by team_member_add alongside
# members_with_roles). This avoids a full scan of all team rows.
user_record: prisma_models.LiteLLM_UserTable | None = await prisma_client.db.litellm_usertable.find_unique(
user_record: Final = await _user_table(prisma_client).find_unique(
where={"user_id": user_api_key_dict.user_id},
)
user_team_ids: Sequence[str] = user_record.teams if user_record is not None and user_record.teams else []
user_team_ids: list[str] = user_record.teams if user_record is not None and user_record.teams else []
projects = await prisma_client.db.litellm_projecttable.find_many(
projects = await _project_table(prisma_client).find_many(
where={"team_id": {"in": user_team_ids}},
include={"litellm_budget_table": True, "object_permission": True},
)

45
enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py
View file

@ -10,7 +10,8 @@ All /vector_store management endpoints
import copy
import json
from typing import List, Optional
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, List, Optional, Protocol
from fastapi import APIRouter, Depends, HTTPException
@ -32,9 +33,35 @@ from litellm.types.vector_stores import (
)
from litellm.vector_stores.vector_store_registry import VectorStoreRegistry
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
router = APIRouter()
class ManagedVectorStoreRow(Protocol):
"""A ``litellm_managedvectorstorestable`` row as returned by Prisma."""
def model_dump(self) -> LiteLLM_ManagedVectorStore: ...
class ManagedVectorStoreTable(Protocol):
"""The Prisma actions namespace for ``litellm_managedvectorstorestable``."""
async def find_unique(self, where: Mapping[str, str | None]) -> ManagedVectorStoreRow | None: ...
async def create(self, data: Mapping[str, object]) -> ManagedVectorStoreRow: ...
async def delete(self, where: Mapping[str, str | None]) -> ManagedVectorStoreRow | None: ...
async def update(self, where: Mapping[str, str | None], data: Mapping[str, object]) -> ManagedVectorStoreRow: ...
def managed_vector_store_table(prisma_client: "PrismaClient") -> ManagedVectorStoreTable:
"""The Prisma table actions for managed vector stores, behind a typed surface."""
return prisma_client.db.litellm_managedvectorstorestable
########################################################
# Management Endpoints
########################################################
@ -66,7 +93,7 @@ async def new_vector_store(
try:
# Check if vector store already exists
existing_vector_store = (
await prisma_client.db.litellm_managedvectorstorestable.find_unique(
await managed_vector_store_table(prisma_client).find_unique(
where={"vector_store_id": vector_store.get("vector_store_id")}
)
)
@ -92,7 +119,7 @@ async def new_vector_store(
del vector_store["litellm_params"]
_new_vector_store = (
await prisma_client.db.litellm_managedvectorstorestable.create(
await managed_vector_store_table(prisma_client).create(
data={
**vector_store,
"litellm_params": litellm_params_json,
@ -213,7 +240,7 @@ async def delete_vector_store(
try:
# Check if vector store exists
existing_vector_store = (
await prisma_client.db.litellm_managedvectorstorestable.find_unique(
await managed_vector_store_table(prisma_client).find_unique(
where={"vector_store_id": data.vector_store_id}
)
)
@ -224,7 +251,7 @@ async def delete_vector_store(
)
# Delete vector store
await prisma_client.db.litellm_managedvectorstorestable.delete(
await managed_vector_store_table(prisma_client).delete(
where={"vector_store_id": data.vector_store_id}
)
@ -288,7 +315,7 @@ async def get_vector_store_info(
return {"vector_store": vector_store_pydantic_obj}
vector_store = (
await prisma_client.db.litellm_managedvectorstorestable.find_unique(
await managed_vector_store_table(prisma_client).find_unique(
where={"vector_store_id": data.vector_store_id}
)
)
@ -298,7 +325,7 @@ async def get_vector_store_info(
detail=f"Vector store with ID {data.vector_store_id} not found",
)
vector_store_dict = vector_store.model_dump() # type: ignore[attr-defined]
vector_store_dict = vector_store.model_dump()
return {"vector_store": vector_store_dict}
except Exception as e:
verbose_proxy_logger.exception(f"Error getting vector store info: {str(e)}")
@ -322,13 +349,13 @@ async def update_vector_store(
try:
update_data = data.model_dump(exclude_unset=True)
vector_store_id = update_data.pop("vector_store_id")
vector_store_id: Final[str] = update_data.pop("vector_store_id")
if update_data.get("vector_store_metadata") is not None:
update_data["vector_store_metadata"] = safe_dumps(
update_data["vector_store_metadata"]
)
updated = await prisma_client.db.litellm_managedvectorstorestable.update(
updated = await managed_vector_store_table(prisma_client).update(
where={"vector_store_id": vector_store_id},
data=update_data,
)

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.54"
version = "0.1.62"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.54"
version = "0.1.62"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

4
gateway/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

1
gateway/routes/allowlist.py
View file

@ -83,6 +83,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
"/azure_ai/",
"/aws/",
"/bedrock/",
"/comprehendmedical",
"/cohere/",
"/gemini/",
"/google/",

2
helm/litellm-helm/Chart.yaml
View file

@ -18,7 +18,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 1.1.1
version: 1.1.2
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to

2
helm/litellm-helm/README.md
View file

@ -29,7 +29,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A |
| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` |
| `image.repository` | LiteLLM Proxy image repository | `docker.litellm.ai/berriai/litellm` |
| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` |
| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` |
| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` |
| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` |

7
helm/litellm-helm/templates/deployment.yaml
View file

@ -100,6 +100,13 @@ spec:
- name: DATABASE_URL
value: {{ .Values.db.url | quote }}
{{- end }}
{{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }}
- name: DATABASE_READER_HOST
valueFrom:
secretKeyRef:
name: {{ .Values.db.secret.name }}
key: {{ .Values.db.secret.readReplicaEndpointKey }}
{{- end }}
{{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
- name: DATABASE_URL_READ_REPLICA
valueFrom:

7
helm/litellm-helm/templates/migrations-job.yaml
View file

@ -105,6 +105,10 @@ spec:
{{- toYaml . | nindent 8 }}
{{- end }}
restartPolicy: OnFailure
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
@ -115,4 +119,7 @@ spec:
{{- end }}
ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
backoffLimit: {{ .Values.migrationJob.backoffLimit }}
{{- with .Values.migrationJob.activeDeadlineSeconds }}
activeDeadlineSeconds: {{ . }}
{{- end }}
{{- end }}

104
helm/litellm-helm/tests/deployment_tests.yaml
View file

@ -15,7 +15,7 @@ tests:
pattern: -litellm$
- equal:
path: spec.template.spec.containers[0].image
value: ghcr.io/berriai/litellm-database:test
value: ghcr.io/berriai/litellm:test
- it: should work with tolerations
template: deployment.yaml
set:
@ -80,6 +80,96 @@ tests:
secretKeyRef:
name: my-secret
key: my-key
- it: should inject DATABASE_READER_HOST from readReplicaEndpointKey before DATABASE_URL_READ_REPLICA
template: deployment.yaml
set:
db:
deployStandalone: false
useExisting: true
secret:
name: postgres
usernameKey: username
passwordKey: password
readReplicaEndpointKey: reader-host
readReplicaUrl: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)?sslmode=require
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_READER_HOST
valueFrom:
secretKeyRef:
name: postgres
key: reader-host
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL_READ_REPLICA
value: postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)?sslmode=require
# $(VAR) interpolation only resolves vars defined EARLIER in the env
# array, so the reader host must precede the composed URL
- equal:
path: spec.template.spec.containers[0].env[7].name
value: DATABASE_READER_HOST
- equal:
path: spec.template.spec.containers[0].env[8].name
value: DATABASE_URL_READ_REPLICA
- it: should omit reader host when readReplicaUrl is unset
template: deployment.yaml
set:
db:
deployStandalone: false
useExisting: true
secret:
name: postgres
usernameKey: username
passwordKey: password
readReplicaEndpointKey: reader-host
asserts:
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_READER_HOST
valueFrom:
secretKeyRef:
name: postgres
key: reader-host
- it: should prefer readReplicaUrlKey over readReplicaEndpointKey composition
template: deployment.yaml
set:
db:
useExisting: true
secret:
name: postgres
usernameKey: username
passwordKey: password
readReplicaUrlKey: reader-url
readReplicaEndpointKey: reader-host
readReplicaUrl: postgresql://ignored
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL_READ_REPLICA
valueFrom:
secretKeyRef:
name: postgres
key: reader-url
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_URL_READ_REPLICA
value: postgresql://ignored
# the unused reader-host secret ref must be suppressed so a missing
# key can't fail pod creation
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_READER_HOST
valueFrom:
secretKeyRef:
name: postgres
key: reader-host
- it: should work with extraEnvVars
template: deployment.yaml
set:
@ -337,7 +427,7 @@ tests:
template: deployment.yaml
set:
image:
repository: ghcr.io/berriai/litellm-database
repository: ghcr.io/berriai/litellm
tag: test
extraInitContainers:
- name: init-tpl
@ -348,7 +438,7 @@ tests:
path: spec.template.spec.initContainers
content:
name: init-tpl
image: "ghcr.io/berriai/litellm-database:test"
image: "ghcr.io/berriai/litellm:test"
command: ["echo", "hello"]
- it: should work with extraContainers
template: deployment.yaml
@ -366,7 +456,7 @@ tests:
template: deployment.yaml
set:
image:
repository: ghcr.io/berriai/litellm-database
repository: ghcr.io/berriai/litellm
tag: test
extraContainers:
- name: sidecar-tpl
@ -376,12 +466,12 @@ tests:
path: spec.template.spec.containers
content:
name: sidecar-tpl
image: "ghcr.io/berriai/litellm-database:test"
image: "ghcr.io/berriai/litellm:test"
- it: should support tpl in podAnnotations
template: deployment.yaml
set:
image:
repository: ghcr.io/berriai/litellm-database
repository: ghcr.io/berriai/litellm
tag: test
# Mirrors the real-world scenario this feature unblocks:
# user disables the built-in ConfigMap (and its built-in checksum/config
@ -398,7 +488,7 @@ tests:
value: "test"
- equal:
path: spec.template.metadata.annotations["example.com/some-key"]
value: "ghcr.io/berriai/litellm-database"
value: "ghcr.io/berriai/litellm"
- equal:
path: spec.template.metadata.annotations["example.com/literal"]
value: "plain-string-value"

60
helm/litellm-helm/tests/migrations-job_tests.yaml
View file

@ -208,7 +208,7 @@ tests:
template: migrations-job.yaml
set:
image:
repository: ghcr.io/berriai/litellm-database
repository: ghcr.io/berriai/litellm
tag: test
migrationJob:
enabled: true
@ -221,7 +221,7 @@ tests:
path: spec.template.spec.initContainers
content:
name: init-tpl
image: "ghcr.io/berriai/litellm-database:test"
image: "ghcr.io/berriai/litellm:test"
command: ["echo", "hello"]
- it: should work with extraContainers
template: migrations-job.yaml
@ -241,7 +241,7 @@ tests:
template: migrations-job.yaml
set:
image:
repository: ghcr.io/berriai/litellm-database
repository: ghcr.io/berriai/litellm
tag: test
migrationJob:
enabled: true
@ -253,7 +253,7 @@ tests:
path: spec.template.spec.containers
content:
name: sidecar-tpl
image: "ghcr.io/berriai/litellm-database:test"
image: "ghcr.io/berriai/litellm:test"
- it: should render the pod-level securityContext from podSecurityContext
template: migrations-job.yaml
set:
@ -290,3 +290,55 @@ tests:
value:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
- it: should schedule onto the same nodes as the gateway
template: migrations-job.yaml
set:
migrationJob:
enabled: true
nodeSelector:
karpenter.sh/nodepool: litellm-e2e
tolerations:
- key: workload
operator: Equal
value: litellm-e2e
effect: NoSchedule
asserts:
- equal:
path: spec.template.spec.nodeSelector
value:
karpenter.sh/nodepool: litellm-e2e
- equal:
path: spec.template.spec.tolerations
value:
- key: workload
operator: Equal
value: litellm-e2e
effect: NoSchedule
- it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever
set:
migrationJob:
enabled: true
asserts:
- equal:
path: spec.activeDeadlineSeconds
value: 1800
- it: honours an operator-supplied deadline
set:
migrationJob:
enabled: true
activeDeadlineSeconds: 600
asserts:
- equal:
path: spec.activeDeadlineSeconds
value: 600
- it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour
set:
migrationJob:
enabled: true
activeDeadlineSeconds: null
asserts:
- notExists:
path: spec.activeDeadlineSeconds

20
helm/litellm-helm/values.yaml
View file

@ -6,8 +6,9 @@ replicaCount: 1
# numWorkers: 2
image:
# Use "ghcr.io/berriai/litellm-database" for optimized image with database
repository: ghcr.io/berriai/litellm-database
# Bundles the prisma CLI and engines, which is what lets the migrations job
# and the proxy's own schema check run without network access.
repository: ghcr.io/berriai/litellm
pullPolicy: Always
# Overrides the image tag whose default is the chart appVersion.
# tag: "latest"
@ -276,6 +277,14 @@ db:
# written to db.readReplicaUrl ends up visible in the rendered pod spec
# and the Helm release secret.
readReplicaUrlKey: ""
# Optional: when set, a DATABASE_READER_HOST env var is sourced from this
# secret key, so db.readReplicaUrl can compose the reader URL from
# individual secret components, e.g.
# postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_READER_HOST):5432/$(DATABASE_NAME)
# Use this when your secret store holds the bare reader hostname rather
# than a full connection URL. Only takes effect when readReplicaUrl is
# set; ignored when readReplicaUrlKey is set.
readReplicaEndpointKey: ""
# Optional read-replica routing. When set, the proxy sends read-only
# queries (find_*, count, group_by, query_raw/_first) to this URL while
@ -427,6 +436,13 @@ migrationJob:
enabled: true # Enable or disable the schema migration Job
retries: 3 # Number of retries for the Job in case of failure
backoffLimit: 4 # Backoff limit for Job restarts
# Wall-clock budget for the whole Job, shared across every `backoffLimit`
# retry rather than granted per attempt. Without it a migration that blocks
# on the database never fails, and when the Helm hook is enabled the release
# waits on it forever: `helm upgrade` and any GitOps controller driving it
# stop reconciling the whole chart until someone deletes the Job by hand.
# Set to null to opt out and restore the unbounded behaviour.
activeDeadlineSeconds: 1800
disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
# Optional service account for the migration job.
# Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true.

26
helm/litellm/templates/_helpers.tpl
View file

@ -213,18 +213,21 @@ whenever the password contains a URL-reserved character (@, /, ?, %, +,
When `database.writer.useIAMAuth: true`, the chart injects
IAM_TOKEN_DB_AUTH=true and omits DATABASE_PASSWORD — the entrypoint mints
the URL from DATABASE_HOST/PORT/USER/NAME plus a short-lived IAM token
instead of a static password.
the URL from DATABASE_HOST/PORT/USER/NAME plus a short-lived AWS RDS IAM
token instead of a static password. `database.writer.useAzureEntraAuth: true`
does the same with AZURE_POSTGRESQL_AUTH=true and a Microsoft Entra ID token,
for Azure Database for PostgreSQL. The two are mutually exclusive.
The read replica is opt-in via `database.reader.host`. The chart emits
DATABASE_HOST_READ_REPLICA / DATABASE_PORT_READ_REPLICA /
DATABASE_NAME_READ_REPLICA (+ DATABASE_SCHEMA_READ_REPLICA) for both auth
modes, plus DATABASE_USER_READ_REPLICA / DATABASE_PASSWORD_READ_REPLICA for
password auth. When `database.reader.useIAMAuth: true` it omits
password auth. When `database.reader.useIAMAuth: true` (or
`database.reader.useAzureEntraAuth: true`) it omits
DATABASE_PASSWORD_READ_REPLICA and the entrypoint mints the reader URL the
same way. Reader IAM only takes effect when the writer also uses IAM auth
(the proxy gates URL minting on IAM_TOKEN_DB_AUTH, which only the writer
sets).
same way. Reader token auth only takes effect when the writer uses the same
token source, since the proxy gates URL minting on the single global
IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets.
*/}}
{{- define "litellm.serverEnv" -}}
{{- $root := .root -}}
@ -254,9 +257,15 @@ sets).
- name: DATABASE_SCHEMA
value: {{ .schema | quote }}
{{- end }}
{{- if and .useIAMAuth .useAzureEntraAuth }}
{{- fail "database.writer.useIAMAuth and database.writer.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source" }}
{{- end }}
{{- if .useIAMAuth }}
- name: IAM_TOKEN_DB_AUTH
value: "true"
{{- else if .useAzureEntraAuth }}
- name: AZURE_POSTGRESQL_AUTH
value: "true"
{{- else }}
- name: DATABASE_PASSWORD
valueFrom:
@ -270,6 +279,9 @@ sets).
{{- if and .useIAMAuth (not $root.Values.database.writer.useIAMAuth) }}
{{- fail "database.reader.useIAMAuth requires database.writer.useIAMAuth: true (the proxy gates IAM URL minting on IAM_TOKEN_DB_AUTH, which is only set by the writer)" }}
{{- end }}
{{- if and .useAzureEntraAuth (not $root.Values.database.writer.useAzureEntraAuth) }}
{{- fail "database.reader.useAzureEntraAuth requires database.writer.useAzureEntraAuth: true (the proxy gates Entra URL minting on AZURE_POSTGRESQL_AUTH, which is only set by the writer)" }}
{{- end }}
- name: DATABASE_HOST_READ_REPLICA
value: {{ .host | quote }}
- name: DATABASE_PORT_READ_REPLICA
@ -280,7 +292,7 @@ sets).
- name: DATABASE_SCHEMA_READ_REPLICA
value: {{ .schema | quote }}
{{- end }}
{{- if .useIAMAuth }}
{{- if or .useIAMAuth .useAzureEntraAuth }}
{{- if .passwordSecret.name }}
- name: DATABASE_USER_READ_REPLICA
valueFrom:

4
helm/litellm/templates/backend/deployment.yaml
View file

@ -81,6 +81,10 @@ spec:
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.backend.startupProbe }}
startupProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.backend.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}

4
helm/litellm/templates/backend/hpa.yaml
View file

@ -30,4 +30,8 @@ spec:
type: Utilization
averageUtilization: {{ .Values.backend.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- with .Values.backend.hpa.behavior }}
behavior:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

4
helm/litellm/templates/gateway/deployment.yaml
View file

@ -83,6 +83,10 @@ spec:
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.startupProbe }}
startupProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.gateway.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}

4
helm/litellm/templates/gateway/hpa.yaml
View file

@ -30,4 +30,8 @@ spec:
type: Utilization
averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- with .Values.gateway.hpa.behavior }}
behavior:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

150
helm/litellm/templates/ingress.yaml
View file

@ -5,6 +5,41 @@
{{- $gatewayPort := .Values.gateway.service.port -}}
{{- $backendPort := .Values.backend.service.port -}}
{{- $uiPort := .Values.ui.service.port -}}
{{/*
Backends addressable from ingress.extraPaths, keyed by the `service` field.
*/}}
{{- $extraPathBackends := dict
"gateway" (dict "name" $gatewayName "port" $gatewayPort)
"backend" (dict "name" $backendName "port" $backendPort)
"ui" (dict "name" $uiName "port" $uiPort)
-}}
{{/*
UI paths (Next.js static export).
/ui/* is where the SPA serves its login + dashboard routes (e.g. /ui/login).
Without it, /ui/* falls into the catch-all → backend → 404.
The App Router (output: "export", basePath: "") emits the RSC/flight payload
for every route as a ROOT-level <route>.txt (/index.txt, /teams.txt,
/__next._tree.txt, ...). The client router fetches these on every soft
navigation / prefetch as <route>.txt?_rsc=<hash> (the query string is
irrelevant to path matching). They are not under /ui, /_next, or
/litellm-asset-prefix, so without /*.txt they fall to the backend catch-all
→ 404 → client-side navigation never settles and the login flow spins in an
infinite redirect loop (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt
from the export; the rule only routes the request to it. Needs an ingress
controller whose ImplementationSpecific path is a wildcard pattern
(AWS ALB: `*` = 0+ chars); this chart targets the AWS Load Balancer
Controller.
*/}}
{{- $uiPaths := list
(dict "path" "/" "pathType" "Exact")
(dict "path" "/favicon.ico" "pathType" "Exact")
(dict "path" "/litellm-asset-prefix" "pathType" "Prefix")
(dict "path" "/_next" "pathType" "Prefix")
(dict "path" "/ui" "pathType" "Prefix")
(dict "path" "/*.txt" "pathType" "ImplementationSpecific")
-}}
{{/*
Gateway data-plane prefixes — must mirror gateway/routes/allowlist.py.
Versioned paths are listed explicitly to avoid routing management routes
@ -24,7 +59,7 @@
"/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
"/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
"/v1beta" "/interactions"
"/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/cohere" "/gemini" "/google"
"/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/comprehendmedical" "/cohere" "/gemini" "/google"
"/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm"
"/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough"
"/toolset"
@ -39,6 +74,21 @@
routes at startup -> 404. So /test is rendered as a standalone Exact path
and /test/* falls through to the backend catch-all.
*/}}
{{/*
Every "<path>|<pathType>" this template renders on its own. An
ingress.extraPaths entry that repeats one of these is rejected: duplicates
in a single rule are resolved by position or by controller-specific tie
breaking, so the operator entry could take over a built-in route (an entry
at "/" Prefix would swallow the whole backend management API) instead of
adding to it.
*/}}
{{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}}
{{- range $uiPaths }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" .path .pathType) }}
{{- end }}
{{- range $gatewayPrefixes }}
{{- $builtinPathKeys = append $builtinPathKeys (printf "%s|Prefix" .) }}
{{- end }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
@ -64,65 +114,15 @@ spec:
http:
paths:
# --- UI (Next.js static export) ---
- path: /
pathType: Exact
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /favicon.ico
pathType: Exact
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /litellm-asset-prefix
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
- path: /_next
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# /ui/* is where the Next.js SPA serves its login + dashboard
# routes (e.g. /ui/login). Without this, /ui/* falls into the
# catch-all → backend → 404.
- path: /ui
pathType: Prefix
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
# Next.js App Router (output: "export", basePath: "") emits the
# RSC/flight payload for every route as a ROOT-level <route>.txt
# (/index.txt, /teams.txt, /__next._tree.txt, ...). The client
# router fetches these on every soft navigation / prefetch as
# <route>.txt?_rsc=<hash> (the query string is irrelevant to path
# matching). They are not under /ui, /_next, or
# /litellm-asset-prefix, so without this rule they fall to the
# backend catch-all → 404 → client-side navigation never settles
# and the login flow spins in an infinite redirect loop
# (/ ⇄ /ui/login). ui/nginx.conf already serves *.txt from the
# export; this rule only routes the request to it. Needs an
# ingress controller whose ImplementationSpecific path is a
# wildcard pattern (AWS ALB: `*` = 0+ chars); this chart targets
# the AWS Load Balancer Controller.
- path: /*.txt
pathType: ImplementationSpecific
{{- range $uiPaths }}
- path: {{ .path }}
pathType: {{ .pathType }}
backend:
service:
name: {{ $uiName }}
port:
number: {{ $uiPort }}
{{- end }}
# --- Gateway data plane ---
# Exact /test only (see the $gatewayPrefixes comment above);
# /test/* MCP management endpoints fall to the backend catch-all.
@ -142,6 +142,46 @@ spec:
port:
number: {{ $gatewayPort }}
{{- end }}
{{- /*
--- Operator-supplied extra paths (ingress.extraPaths) ---
Rendered after every built-in path so an entry can never take
precedence over a default, and before the backend catch-all.
Position only decides the match on controllers that honour manifest
order: the AWS Load Balancer Controller this chart targets sorts
Exact paths first and Prefix paths longest-first, but keeps
ImplementationSpecific paths in manifest order, which is what the
/*.txt rule above already depends on.
*/}}
{{- range $idx, $extra := .Values.ingress.extraPaths }}
{{- if not (kindIs "map" $extra) }}
{{- fail (printf "ingress.extraPaths[%d]: each entry must be a mapping with a 'path' key" $idx) }}
{{- end }}
{{- if not $extra.path }}
{{- fail (printf "ingress.extraPaths[%d]: 'path' is required" $idx) }}
{{- end }}
{{- $service := $extra.service | default "gateway" }}
{{- $target := get $extraPathBackends $service }}
{{- if not $target }}
{{- fail (printf "ingress.extraPaths[%d] (path %s): unknown service %q, expected one of backend, gateway, ui" $idx $extra.path $service) }}
{{- end }}
{{- $pathType := $extra.pathType | default "Prefix" }}
{{- if not (has $pathType (list "Prefix" "Exact" "ImplementationSpecific")) }}
{{- fail (printf "ingress.extraPaths[%d] (path %s): unknown pathType %q, expected one of Exact, ImplementationSpecific, Prefix" $idx $extra.path $pathType) }}
{{- end }}
{{- if eq $extra.path "/" }}
{{- fail (printf "ingress.extraPaths[%d]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture" $idx) }}
{{- end }}
{{- if has (printf "%s|%s" $extra.path $pathType) $builtinPathKeys }}
{{- fail (printf "ingress.extraPaths[%d]: path %s with pathType %s is already routed by this chart, and a duplicate would take it over rather than add to it" $idx $extra.path $pathType) }}
{{- end }}
- path: {{ $extra.path | quote }}
pathType: {{ $pathType }}
backend:
service:
name: {{ $target.name }}
port:
number: {{ $target.port }}
{{- end }}
# --- Catch-all → backend (management API: /key/*, /user/*, /team/*, ...) ---
- path: /
pathType: Prefix

3
helm/litellm/templates/migrations-job.yaml
View file

@ -21,6 +21,9 @@ metadata:
spec:
backoffLimit: {{ .Values.migrationJob.backoffLimit }}
ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
{{- with .Values.migrationJob.activeDeadlineSeconds }}
activeDeadlineSeconds: {{ . }}
{{- end }}
template:
metadata:
{{- /* The Job's selector is generated by the controller rather than

4
helm/litellm/templates/ui/deployment.yaml
View file

@ -69,6 +69,10 @@ spec:
readinessProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.ui.startupProbe }}
startupProbe:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.ui.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}

4
helm/litellm/templates/ui/hpa.yaml
View file

@ -30,4 +30,8 @@ spec:
type: Utilization
averageUtilization: {{ .Values.ui.hpa.targetMemoryUtilizationPercentage }}
{{- end }}
{{- with .Values.ui.hpa.behavior }}
behavior:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

116
helm/litellm/tests/database_auth_tests.yaml Normal file
View file

@ -0,0 +1,116 @@
suite: test database token auth env vars
templates:
- gateway/deployment.yaml
- gateway/configmap.yaml
- backend/deployment.yaml
- backend/configmap.yaml
values:
- ./values/required.yaml
tests:
- it: writer emits DATABASE_PASSWORD and no token toggle by default
template: gateway/deployment.yaml
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PASSWORD
valueFrom:
secretKeyRef:
name: litellm-writer-secret
key: password
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: IAM_TOKEN_DB_AUTH
value: "true"
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: AZURE_POSTGRESQL_AUTH
value: "true"
any: true
- it: writer emits AZURE_POSTGRESQL_AUTH and omits DATABASE_PASSWORD under Entra auth
template: gateway/deployment.yaml
set:
database.writer.useAzureEntraAuth: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: AZURE_POSTGRESQL_AUTH
value: "true"
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PASSWORD
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: IAM_TOKEN_DB_AUTH
value: "true"
any: true
- it: backend gets the same Entra toggle as the gateway
template: backend/deployment.yaml
set:
database.writer.useAzureEntraAuth: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: AZURE_POSTGRESQL_AUTH
value: "true"
any: true
- it: writer rejects both token sources at once
template: gateway/deployment.yaml
set:
database.writer.useIAMAuth: true
database.writer.useAzureEntraAuth: true
asserts:
- failedTemplate:
errorMessage: "database.writer.useIAMAuth and database.writer.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source"
- it: reader Entra auth without writer Entra auth is rejected
template: gateway/deployment.yaml
set:
database.reader.host: reader.example.com
database.reader.dbname: litellm
database.reader.useAzureEntraAuth: true
asserts:
- failedTemplate:
errorMessage: "database.reader.useAzureEntraAuth requires database.writer.useAzureEntraAuth: true (the proxy gates Entra URL minting on AZURE_POSTGRESQL_AUTH, which is only set by the writer)"
- it: reader under Entra auth omits DATABASE_PASSWORD_READ_REPLICA
template: gateway/deployment.yaml
set:
database.writer.useAzureEntraAuth: true
database.reader.host: reader.example.com
database.reader.dbname: litellm
database.reader.useAzureEntraAuth: true
asserts:
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_HOST_READ_REPLICA
value: reader.example.com
any: true
- contains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_USER_READ_REPLICA
valueFrom:
secretKeyRef:
name: litellm-reader-secret
key: username
any: true
- notContains:
path: spec.template.spec.containers[0].env
content:
name: DATABASE_PASSWORD_READ_REPLICA
any: true

58
helm/litellm/tests/hpa_behavior_tests.yaml Normal file
View file

@ -0,0 +1,58 @@
suite: test HPA scaling behavior passthrough
templates:
- gateway/hpa.yaml
- backend/hpa.yaml
- ui/hpa.yaml
values:
- ./values/required.yaml
tests:
- it: HPA omits spec.behavior by default, so Kubernetes' default scaling applies
templates:
- gateway/hpa.yaml
- backend/hpa.yaml
asserts:
- isKind:
of: HorizontalPodAutoscaler
- notExists:
path: spec.behavior
- it: gateway HPA renders spec.behavior verbatim when configured
template: gateway/hpa.yaml
set:
gateway.hpa.behavior:
scaleDown:
stabilizationWindowSeconds: 300
policies:
- { type: Percent, value: 50, periodSeconds: 60 }
scaleUp:
stabilizationWindowSeconds: 0
selectPolicy: Max
policies:
- { type: Percent, value: 100, periodSeconds: 30 }
- { type: Pods, value: 2, periodSeconds: 30 }
asserts:
- equal:
path: spec.behavior
value:
scaleDown:
stabilizationWindowSeconds: 300
policies:
- { type: Percent, value: 50, periodSeconds: 60 }
scaleUp:
stabilizationWindowSeconds: 0
selectPolicy: Max
policies:
- { type: Percent, value: 100, periodSeconds: 30 }
- { type: Pods, value: 2, periodSeconds: 30 }
- it: behavior passthrough works on every autoscaled component (ui parity)
template: ui/hpa.yaml
set:
ui.hpa.enabled: true
ui.hpa.behavior:
scaleUp:
stabilizationWindowSeconds: 0
asserts:
- equal:
path: spec.behavior.scaleUp.stabilizationWindowSeconds
value: 0

317
helm/litellm/tests/ingress_extra_paths_tests.yaml Normal file
View file

@ -0,0 +1,317 @@
suite: test ingress.extraPaths
templates:
- ingress.yaml
values:
- ./values/required.yaml
tests:
- it: renders nothing extra between the built-in gateway prefixes and the backend catch-all when unset
set:
ingress.enabled: true
asserts:
- equal:
path: spec.rules[0].http.paths[-1]
value:
path: /
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-backend
port:
number: 4001
- equal:
path: spec.rules[0].http.paths[-2]
value:
path: /metrics
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- it: routes an extra path to the gateway by default, immediately before the backend catch-all
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
asserts:
- equal:
path: spec.rules[0].http.paths[-2]
value:
path: /watsonx
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- equal:
path: spec.rules[0].http.paths[-1]
value:
path: /
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-backend
port:
number: 4001
- it: keeps every built-in path when extra paths are supplied
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
asserts:
- contains:
path: spec.rules[0].http.paths
content:
path: /
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- contains:
path: spec.rules[0].http.paths
content:
path: /ui
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- contains:
path: spec.rules[0].http.paths
content:
path: /test
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- contains:
path: spec.rules[0].http.paths
content:
path: /v1/chat
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- contains:
path: spec.rules[0].http.paths
content:
path: /vertex_ai
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- it: renders every entry in order and honours the service and pathType selectors
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
service: gateway
- path: /my-passthrough
pathType: Exact
service: backend
- path: /brand.txt
pathType: ImplementationSpecific
service: ui
asserts:
- equal:
path: spec.rules[0].http.paths[-4]
value:
path: /watsonx
pathType: Prefix
backend:
service:
name: RELEASE-NAME-litellm-gateway
port:
number: 4000
- equal:
path: spec.rules[0].http.paths[-3]
value:
path: /my-passthrough
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-backend
port:
number: 4001
- equal:
path: spec.rules[0].http.paths[-2]
value:
path: /brand.txt
pathType: ImplementationSpecific
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- it: addresses the component services by their configured ports
set:
ingress.enabled: true
gateway.service.port: 8000
backend.service.port: 8001
ui.service.port: 8080
ingress.extraPaths:
- path: /watsonx
- path: /my-passthrough
service: backend
- path: /brand.txt
service: ui
asserts:
- equal:
path: spec.rules[0].http.paths[-4].backend.service.port.number
value: 8000
- equal:
path: spec.rules[0].http.paths[-3].backend.service.port.number
value: 8001
- equal:
path: spec.rules[0].http.paths[-2].backend.service.port.number
value: 8080
- it: rejects an entry naming a service the chart does not deploy
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
service: proxy
asserts:
- failedTemplate:
errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown service "proxy", expected one of backend, gateway, ui'
- it: rejects an entry whose pathType is not a kubernetes pathType
set:
ingress.enabled: true
ingress.extraPaths:
- path: /watsonx
pathType: prefix
asserts:
- failedTemplate:
errorMessage: 'ingress.extraPaths[0] (path /watsonx): unknown pathType "prefix", expected one of Exact, ImplementationSpecific, Prefix'
- it: rejects an entry with no path
set:
ingress.enabled: true
ingress.extraPaths:
- service: gateway
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: 'path' is required"
- it: rejects a root entry that would take over the backend catch-all
set:
ingress.enabled: true
ingress.extraPaths:
- path: /
service: gateway
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
- it: rejects a root entry that would take over the UI root
set:
ingress.enabled: true
ingress.extraPaths:
- path: /
pathType: Exact
service: gateway
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
# A root ImplementationSpecific entry duplicates no built-in pair, so the
# duplicate check alone would admit it. It is still dead: the built-in
# Exact / sorts ahead of it on the AWS Load Balancer Controller and claims
# the only request its pattern matches, so it renders and never routes.
- it: rejects a root entry that would render but never match
set:
ingress.enabled: true
ingress.extraPaths:
- path: /
pathType: ImplementationSpecific
service: gateway
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path / is already routed in both directions, Exact to ui and Prefix to backend, so no pathType leaves a request for an entry here to capture"
- it: rejects an entry that would take over a UI prefix
set:
ingress.enabled: true
ingress.extraPaths:
- path: /ui
service: gateway
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path /ui with pathType Prefix is already routed by this chart, and a duplicate would take it over rather than add to it"
- it: rejects an entry that would take over the UI RSC payload rule
set:
ingress.enabled: true
ingress.extraPaths:
- path: /*.txt
pathType: ImplementationSpecific
service: backend
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path /*.txt with pathType ImplementationSpecific is already routed by this chart, and a duplicate would take it over rather than add to it"
- it: rejects an entry that would take over a gateway data-plane prefix
set:
ingress.enabled: true
ingress.extraPaths:
- path: /v1/chat
service: backend
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path /v1/chat with pathType Prefix is already routed by this chart, and a duplicate would take it over rather than add to it"
- it: rejects an entry that would take over the exact /test route
set:
ingress.enabled: true
ingress.extraPaths:
- path: /test
pathType: Exact
service: backend
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: path /test with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it"
- it: allows a built-in path under a different pathType, which is a distinct rule
set:
ingress.enabled: true
ingress.extraPaths:
- path: /ui
pathType: Exact
service: ui
asserts:
- equal:
path: spec.rules[0].http.paths[-2]
value:
path: /ui
pathType: Exact
backend:
service:
name: RELEASE-NAME-litellm-ui
port:
number: 3000
- it: rejects a bare string entry instead of failing on template internals
set:
ingress.enabled: true
ingress.extraPaths:
- /watsonx
asserts:
- failedTemplate:
errorMessage: "ingress.extraPaths[0]: each entry must be a mapping with a 'path' key"

21
helm/litellm/tests/migration_job_tests.yaml
View file

@ -167,3 +167,24 @@ tests:
- equal:
path: spec.template.metadata.labels['app.kubernetes.io/component']
value: batch-migrations
- it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever
asserts:
- equal:
path: spec.activeDeadlineSeconds
value: 1800
- it: honours an operator-supplied deadline
set:
migrationJob.activeDeadlineSeconds: 600
asserts:
- equal:
path: spec.activeDeadlineSeconds
value: 600
- it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour
set:
migrationJob.activeDeadlineSeconds: null
asserts:
- notExists:
path: spec.activeDeadlineSeconds

27
helm/litellm/tests/probe_tests.yaml
View file

@ -104,3 +104,30 @@ tests:
periodSeconds: 15
timeoutSeconds: 4
failureThreshold: 3
- it: no startupProbe by default, so existing installs are unchanged
templates:
- gateway/deployment.yaml
- backend/deployment.yaml
asserts:
- notExists:
path: spec.template.spec.containers[0].startupProbe
- it: startupProbe renders verbatim when configured, gating a slow cold start
template: gateway/deployment.yaml
set:
gateway.startupProbe:
httpGet: { path: /health/readiness, port: http }
failureThreshold: 30
periodSeconds: 10
timeoutSeconds: 5
asserts:
- equal:
path: spec.template.spec.containers[0].startupProbe
value:
httpGet:
path: /health/readiness
port: http
failureThreshold: 30
periodSeconds: 10
timeoutSeconds: 5

Some files were not shown because too many files have changed in this diff Show more