Merge remote-tracking branch 'origin/main' into litellm_cli_login_team_option

.circleci/config.yml

@@ -9,7 +9,7 @@ commands:
     parameters:
       category:
         type: enum
-        enum: ["backend", "client"]
+        enum: ["backend", "client", "provider-harness"]
         default: "backend"
     steps:
       - run:
@@ -147,6 +147,9 @@ commands:
       db_name:
         type: string
         default: circle_test
+      image:
+        type: string
+        default: postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
     steps:
       - run:
           name: Start PostgreSQL
@@ -157,7 +160,7 @@ commands:
               -e POSTGRES_PASSWORD=postgres \
               -e POSTGRES_DB=<< parameters.db_name >> \
               -p 5432:5432 \
-              postgres:14@sha256:6a70deda415ec296f977890e11aba04a0db9f632a362e3fce45e845e3db74f26
+              << parameters.image >>
       - wait_for_service:
           url: tcp://localhost:5432
           timeout: "60"
@@ -254,7 +257,7 @@ commands:
       - install_rust
       - restore_cache:
           keys:
-            - v1-uv-cache-{{ checksum "uv.lock" }}
+            - v3-integration-uv-cache-{{ checksum "uv.lock" }}
       - run:
           name: Install Dependencies
           command: |
@@ -263,7 +266,7 @@ commands:
       - save_cache:
           paths:
             - ~/.cache/uv
-          key: v1-uv-cache-{{ checksum "uv.lock" }}
+          key: v3-integration-uv-cache-{{ checksum "uv.lock" }}
 
 jobs:
   # Add Windows testing job
@@ -1084,9 +1087,7 @@ jobs:
           name: Run tests
           command: |
             mkdir -p test-results
-            TEST_FILES=$(printf "%s\n%s\n" \
-              "$(circleci tests glob "tests/ocr_tests/**/test_*.py")" \
-              "tests/test_litellm/ocr/test_rust_bridge.py")
+            TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
@@ -2914,7 +2915,106 @@ jobs:
               exit 1
             fi
 
+  provider_replay_harness:
+    docker:
+      - *python312_image
+      - image: redis@sha256:e2debfb7956fa12c7ddc79d7e645c8cf26b30c99a6e9161ea9bf4171e1668a5f
+    working_directory: ~/project
+    resource_class: medium
+    environment:
+      E2E_CACHE_TEST_REDIS_URL: redis://127.0.0.1:6379/0
+      E2E_PROVIDER_CACHE: "0"
+      E2E_FIXTURE_MODE: live
+    steps:
+      - checkout
+      - skip_if_unrelated_changes:
+          category: provider-harness
+      - setup_litellm_test_deps
+      - wait_for_service:
+          url: tcp://localhost:6379
+      - run:
+          name: Test provider capture and replay harness
+          command: |
+            mkdir -p test-results/provider-replay-harness
+            uv run --no-sync pytest -q --noconftest -o addopts= -o pythonpath=tests/e2e -p no:rerunfailures \
+              --junitxml=test-results/provider-replay-harness/junit.xml \
+              tests/e2e/test_provider_edge.py tests/e2e/test_fixture_bundle.py \
+              tests/e2e/test_fixture_canonical.py tests/e2e/test_fixture_mode.py \
+              tests/code_coverage_tests/test_provider_replay_harness.py \
+              tests/code_coverage_tests/test_provider_cache.py
+      - store_test_results:
+          path: test-results/provider-replay-harness
+
+  integration_contracts:
+    parameters:
+      suite:
+        type: string
+    machine:
+      image: ubuntu-2204:2024.04.1
+    resource_class: large
+    working_directory: ~/project
+    steps:
+      - setup_litellm_test_deps
+      - when:
+          condition:
+            equal: [browser, << parameters.suite >>]
+          steps:
+            - install_node
+            - restore_cache:
+                keys:
+                  - integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
+            - run:
+                name: Install locked browser dependencies
+                command: |
+                  cd ui/litellm-dashboard
+                  npm ci
+                  cd ../../tests/e2e/ui
+                  npm ci
+                  sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l \
+                    timeout --signal=TERM --kill-after=20s 6m node node_modules/@playwright/test/cli.js install-deps chromium
+                  timeout --signal=TERM --kill-after=20s 3m node node_modules/@playwright/test/cli.js install chromium
+            - save_cache:
+                key: integration-ui-v3-{{ checksum "ui/litellm-dashboard/package-lock.json" }}-{{ checksum "tests/e2e/ui/package-lock.json" }}
+                paths:
+                  - ~/.npm
+                  - ~/.cache/ms-playwright
+            - run:
+                name: Build the candidate dashboard
+                command: cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build
+      - start_postgres:
+          image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+      - start_redis
+      - run:
+          name: Run owned integration contracts
+          command: bash .circleci/scripts/run_integration.sh << parameters.suite >>
+          no_output_timeout: 15m
+      - run:
+          name: Stop owned database and Redis
+          when: always
+          command: |
+            mkdir -p test-results/integration-<< parameters.suite >>
+            docker logs postgres-db > test-results/integration-<< parameters.suite >>/postgres.log 2>&1 || true
+            docker logs redis-cache > test-results/integration-<< parameters.suite >>/redis.log 2>&1 || true
+            docker rm -f postgres-db redis-cache
+            test -z "$(docker ps -aq --filter name=postgres-db --filter name=redis-cache)"
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+
 workflows:
+  integration:
+    jobs:
+      - integration_contracts:
+          name: integration-<< matrix.suite >>
+          matrix:
+            parameters:
+              suite: [management, accounting, database, providers, extensions, sdk, browser]
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
   build_and_test:
     jobs:
       - using_litellm_on_windows:
@@ -2923,6 +3023,7 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - provider_replay_harness
       - base_sdk_install:
           filters: *main_branches
       - local_testing_part1:

.circleci/scripts/classify_changes.sh

@@ -1,22 +1,42 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: classify_changes.sh <backend|client|ui>}"
+category="${1:?usage: classify_changes.sh <backend|client|ui|provider-harness|cost-map-only>}"
 
 has_client=false
 has_backend=false
 has_ci=false
+has_provider_harness=false
+has_cost_map=false
+outside_cost_map_set=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
+  case "$file" in
+    tests/e2e/*/*.py) : ;;
+    tests/e2e/*.py | tests/code_coverage_tests/test_provider_cache.py | tests/code_coverage_tests/test_provider_replay_harness.py | tests/test_litellm/test_circleci_path_filter.py | .circleci/* | pyproject.toml | uv.lock)
+      has_provider_harness=true ;;
+  esac
   case "$file" in
     ui/* | tests/e2e/ui/*) has_client=true ;;
     docs/* | *.md | *.mdx) : ;;
     .github/* | .circleci/*) has_ci=true; has_backend=true ;;
     *) has_backend=true ;;
   esac
+  case "$file" in
+    model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
+      has_cost_map=true ;;
+    tests/test_litellm/* | tests/proxy_unit_tests/*) : ;;
+    *) outside_cost_map_set=true ;;
+  esac
 done
 
 case "$category" in
+  cost-map-only)
+    { [ "$has_cost_map" = true ] && [ "$outside_cost_map_set" = false ]; } && echo run || echo skip
+    ;;
+  provider-harness)
+    [ "$has_provider_harness" = true ] && echo run || echo skip
+    ;;
   backend)
     [ "$has_backend" = true ] && echo run || echo skip
     ;;

.circleci/scripts/path_filter.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: path_filter.sh <backend|client>}"
+category="${1:?usage: path_filter.sh <backend|client|provider-harness>}"
 here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 run_full() {
@@ -36,5 +36,5 @@ if [ "$decision" = run ]; then
   run_full "$category-relevant changes detected"
 fi
 
-echo "path-filter[$category]: only unrelated (docs/client) changes detected; halting job as successful"
+echo "path-filter[$category]: only unrelated changes detected; halting job as successful"
 circleci-agent step halt

.circleci/scripts/run_integration.sh

@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ "${GITHUB_ACTIONS:-}" = true ]; then
+  echo "Integration contracts are owned by CircleCI" >&2
+  exit 1
+fi
+
+suite="${1:?integration suite required}"
+results="test-results/integration-${suite}"
+mkdir -p "$results"
+integration_identity="$(.venv/bin/python -c 'import uuid; print(uuid.uuid4().hex)')"
+upstream_pid=""
+proxy_pid=""
+peer_pid=""
+launched_pid=""
+guard_created=false
+guard_installed=false
+guard6_created=false
+guard6_installed=false
+cleanup() {
+  original_status=$?
+  trap - EXIT INT TERM
+  sudo .venv/bin/python .circleci/scripts/stop_integration_processes.py \
+    "$integration_identity" "$(id -u)" "$proxy_pid" "$peer_pid" "$upstream_pid" \
+    > "$results/process-cleanup.txt" 2>&1 || original_status=1
+  for owned_pid in "$peer_pid" "$proxy_pid" "$upstream_pid"; do
+    if [ -n "$owned_pid" ]; then
+      kill -- "-$owned_pid" 2>/dev/null || true
+      for _ in {1..50}; do
+        kill -0 -- "-$owned_pid" 2>/dev/null || break
+        sleep 0.1
+      done
+      if kill -0 -- "-$owned_pid" 2>/dev/null; then
+        kill -KILL -- "-$owned_pid" 2>/dev/null || true
+        original_status=1
+      fi
+      wait "$owned_pid" 2>/dev/null || true
+    fi
+  done
+  if [ "$guard_installed" = true ]; then
+    sudo iptables -D OUTPUT -m owner --uid-owner "$(id -u)" -j integration_only || original_status=1
+  fi
+  if [ "$guard_created" = true ]; then
+    sudo iptables -F integration_only || original_status=1
+    sudo iptables -X integration_only || original_status=1
+  fi
+  if [ "$guard6_installed" = true ]; then
+    sudo ip6tables -D OUTPUT -m owner --uid-owner "$(id -u)" -j integration_only || original_status=1
+  fi
+  if [ "$guard6_created" = true ]; then
+    sudo ip6tables -F integration_only || original_status=1
+    sudo ip6tables -X integration_only || original_status=1
+  fi
+  printf '%s\n' "$original_status" > "$results/exit-status.txt"
+  exit "$original_status"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+export PATH="$PWD/.venv/bin:$PATH"
+export PYTHONPATH="$PWD:$PWD/tests:$PWD/tests/e2e"
+export DATABASE_URL="postgresql://postgres:postgres@127.0.0.1:5432/circle_test"
+export REDIS_HOST=127.0.0.1 REDIS_PORT=6379
+export LITELLM_MASTER_KEY=sk-integration-master LITELLM_SALT_KEY=sk-integration-salt
+export LITELLM_MODE=PRODUCTION LITELLM_LOCAL_MODEL_COST_MAP=True
+export STORE_MODEL_IN_DB=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1
+export INTEGRATION_PROXY_URL=http://127.0.0.1:4000
+export INTEGRATION_PEER_URL=""
+export INTEGRATION_UPSTREAM_URL=http://127.0.0.1:8190
+export INTEGRATION_MASTER_KEY="$LITELLM_MASTER_KEY"
+export LITELLM_UI_PATH="$PWD/litellm/proxy/_experimental/out"
+if [ "$suite" = browser ]; then
+  export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
+  test -f "$LITELLM_UI_PATH/index.html"
+fi
+export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
+export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
+
+uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
+
+sudo iptables -N integration_only
+guard_created=true
+sudo iptables -A integration_only -o lo -j ACCEPT
+sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
+for service in postgres-db redis-cache; do
+  address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
+  sudo iptables -A integration_only -d "$address" -j ACCEPT
+done
+sudo iptables -A integration_only -j REJECT
+sudo iptables -I OUTPUT 1 -m owner --uid-owner "$(id -u)" -j integration_only
+guard_installed=true
+sudo ip6tables -N integration_only
+guard6_created=true
+sudo ip6tables -A integration_only -o lo -j ACCEPT
+sudo ip6tables -A integration_only -j REJECT
+sudo ip6tables -I OUTPUT 1 -m owner --uid-owner "$(id -u)" -j integration_only
+guard6_installed=true
+
+if curl --noproxy '*' --connect-timeout 2 -s http://198.51.100.1 >/dev/null 2>&1; then
+  echo "Unexpected outbound network access" >&2
+  exit 1
+fi
+sudo iptables -L integration_only -n -v -x > "$results/egress-guard.txt"
+awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/egress-guard.txt"
+
+setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
+  .venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
+upstream_pid=$!
+start_proxy() {
+  local port="$1"
+  local log_name="$2"
+  setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
+    DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
+    LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
+    LITELLM_MODE=PRODUCTION LITELLM_LOCAL_MODEL_COST_MAP=True STORE_MODEL_IN_DB=True \
+    AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
+    .venv/bin/python -m integration._support.proxy --config tests/integration/proxy_config.yaml \
+    --host 127.0.0.1 --port "$port" --num_workers 1 --telemetry False \
+    --use_prisma_db_push --enforce_prisma_migration_check \
+    > "$results/$log_name" 2>&1 &
+  launched_pid=$!
+}
+start_proxy 4000 proxy.log
+proxy_pid="$launched_pid"
+.venv/bin/python .circleci/scripts/wait_integration_services.py
+if [ "$suite" = management ]; then
+  export INTEGRATION_PEER_URL=http://127.0.0.1:4001
+  start_proxy 4001 peer.log
+  peer_pid="$launched_pid"
+  .venv/bin/python .circleci/scripts/wait_integration_services.py
+fi
+
+if [ "$suite" = providers ]; then
+  INTEGRATION_RUN_ID="$integration_identity" .venv/bin/python -m pytest --noconftest -o addopts= \
+    --strict-markers --strict-config -p no:pytest-retry -p no:rerunfailures --timeout=30 \
+    tests/e2e/test_provider_edge.py::TestReplayMode::test_content_drift_returns_the_miss_status_naming_both_keys \
+    tests/e2e/test_provider_edge.py::TestReplayMode::test_exhausted_key_returns_the_miss_status \
+    tests/e2e/test_provider_edge.py::TestReplayLeftover::test_partially_consumed_recording_names_the_leftover \
+    tests/e2e/test_provider_edge.py::TestStreamingFidelity::test_replay_of_a_stream_makes_no_provider_connection \
+    --junitxml="$results/replay-controls.xml"
+fi
+
+if [ "$suite" = browser ]; then
+  export E2E_UI_BASE_URL="$INTEGRATION_PROXY_URL" E2E_UI_ARTIFACT_DIR="$PWD/$results"
+  export INTEGRATION_PYTHON="$PWD/.venv/bin/python"
+  timeout --signal=TERM --kill-after=20s 3m env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
+    INTEGRATION_RUN_ID="$integration_identity" DATABASE_URL="$DATABASE_URL" \
+    INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" INTEGRATION_PYTHON="$INTEGRATION_PYTHON" \
+    E2E_UI_BASE_URL="$E2E_UI_BASE_URL" E2E_UI_ARTIFACT_DIR="$E2E_UI_ARTIFACT_DIR" \
+    LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" CI=true \
+    node tests/e2e/ui/node_modules/@playwright/test/cli.js test --config tests/e2e/ui/integration.config.ts
+  .venv/bin/python .circleci/scripts/verify_integration_browser.py "$results/browser-results.json"
+  exit 0
+fi
+
+timeout --signal=TERM --kill-after=20s 11m env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
+  INTEGRATION_RUN_ID="$integration_identity" \
+  DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
+  INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \
+  INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
+  INTEGRATION_MASTER_KEY="$INTEGRATION_MASTER_KEY" LITELLM_MODE=PRODUCTION \
+  INTEGRATION_SEED="$INTEGRATION_SEED" \
+  INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
+  LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
+  .venv/bin/python tests/integration/run.py "$suite" --results "$results"

.circleci/scripts/stop_integration_processes.py

@@ -0,0 +1,53 @@
+import sys
+from typing import Final
+
+import psutil
+
+
+def is_owned(process: psutil.Process, identity: str, owner_uid: int) -> bool:
+    try:
+        return process.uids().real == owner_uid and process.environ().get("INTEGRATION_RUN_ID") == identity
+    except psutil.NoSuchProcess:
+        return False
+
+
+def owned_processes(identity: str, owner_uid: int) -> tuple[psutil.Process, ...]:
+    return tuple(process for process in psutil.process_iter() if is_owned(process, identity, owner_uid))
+
+
+def main(identity: str, owner_uid: int, root_pids: tuple[int, ...]) -> int:
+    assert owner_uid > 0, "The integration process owner must be a non-root UID"
+    owned: Final = owned_processes(identity, owner_uid)
+    roots: Final = tuple(process for process in owned if process.pid in root_pids)
+    for process in roots:
+        try:
+            process.terminate()
+        except psutil.NoSuchProcess:
+            continue
+    psutil.wait_procs(roots, timeout=30)
+    residual: Final = owned_processes(identity, owner_uid)
+    for process in residual:
+        try:
+            process.terminate()
+        except psutil.NoSuchProcess:
+            continue
+    psutil.wait_procs(residual, timeout=10)
+    remaining: Final = owned_processes(identity, owner_uid)
+    for process in remaining:
+        try:
+            process.kill()
+        except psutil.NoSuchProcess:
+            continue
+    psutil.wait_procs(remaining, timeout=2)
+    survivors: Final = owned_processes(identity, owner_uid)
+    print(
+        f"Owned integration processes: {len(owned)}, roots: {len(roots)}, "
+        f"residual: {len(residual)}, forced: {len(remaining)}, remaining: {len(survivors)}"
+    )
+    for process in remaining:
+        print(f"Forced cleanup was required for PID {process.pid}")
+    return 1 if remaining or survivors else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1], int(sys.argv[2]), tuple(int(value) for value in sys.argv[3:] if value)))

.circleci/scripts/verify_integration_browser.py

@@ -0,0 +1,60 @@
+import json
+import sys
+from pathlib import Path
+from typing import Final
+
+from pydantic import TypeAdapter
+from typing_extensions import NotRequired, ReadOnly, TypedDict
+
+
+class BrowserAttempt(TypedDict):
+    status: ReadOnly[str]
+    retry: ReadOnly[int]
+
+
+class BrowserTest(TypedDict):
+    results: ReadOnly[list[BrowserAttempt]]
+
+
+class BrowserSpec(TypedDict):
+    file: ReadOnly[str]
+    title: ReadOnly[str]
+    tests: ReadOnly[list[BrowserTest]]
+
+
+class BrowserSuite(TypedDict):
+    specs: NotRequired[ReadOnly[list[BrowserSpec]]]
+    suites: NotRequired[ReadOnly[list["BrowserSuite"]]]
+
+
+def main() -> None:
+    result: Final = json.loads(Path(sys.argv[1]).read_text())
+    assert not result.get("errors"), result.get("errors")
+    expected: Final = json.loads(
+        (Path(__file__).resolve().parents[2] / "tests/integration/contracts.json").read_text()
+    )["browser"]
+    assert expected and result["stats"]["expected"] == len(expected)
+    assert all(result["stats"][name] == 0 for name in ("unexpected", "flaky", "skipped"))
+
+    def cases(suite: BrowserSuite) -> tuple[BrowserSpec, ...]:
+        return tuple(suite.get("specs", ())) + tuple(spec for child in suite.get("suites", ()) for spec in cases(child))
+
+    suites: Final = TypeAdapter(list[BrowserSuite]).validate_python(result["suites"], strict=True)
+    specs: Final = tuple(spec for suite in suites for spec in cases(suite))
+    repository: Final = Path(__file__).resolve().parents[2]
+    report_root: Final = Path(result["config"]["rootDir"])
+    assert report_root.is_absolute(), "Playwright rootDir must be explicit"
+    observed: Final = tuple(
+        str((report_root / spec["file"]).resolve().relative_to(repository)) + "::" + spec["title"] for spec in specs
+    )
+    assert sorted(observed) == sorted(expected)
+    for spec in specs:
+        tests: Final = spec["tests"]
+        assert len(tests) == 1 and len(tests[0]["results"]) == 1
+        assert tests[0]["results"][0]["status"] == "passed" and tests[0]["results"][0]["retry"] == 0
+
+    sys.stdout.write("One canonical browser contract passed once without skips or retries\n")
+
+
+if __name__ == "__main__":
+    main()

.circleci/scripts/wait_integration_services.py

@@ -0,0 +1,43 @@
+import os
+import time
+from typing import Final
+
+import httpx
+from redis import Redis
+
+
+def main() -> None:
+    primary: Final = os.environ["INTEGRATION_PROXY_URL"]
+    peer: Final = os.environ.get("INTEGRATION_PEER_URL")
+    proxies: Final = (primary, peer) if peer else (primary,)
+    deadline: Final = time.monotonic() + 90
+    headers: Final = {"Authorization": f"Bearer {os.environ['INTEGRATION_MASTER_KEY']}"}
+    with httpx.Client(trust_env=False, timeout=2) as client, Redis(
+        host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), socket_timeout=2
+    ) as cache:
+        while True:
+            try:
+                ready: Final = (
+                    client.get(f"{os.environ['INTEGRATION_UPSTREAM_URL']}/health").status_code == 200
+                    and all(client.get(f"{url}/health/readiness").status_code == 200 for url in proxies)
+                )
+                if ready:
+                    for url in proxies:
+                        response: Final = client.get(f"{url}/cache/ping", headers=headers)
+                        response.raise_for_status()
+                        result: Final = response.json()
+                        assert result["status"] == "healthy", result
+                        assert result["cache_type"] == "redis", result
+                        assert result["ping_response"] is True, result
+                        assert result["set_cache_response"] == "success", result
+                    if cache.pubsub_numsub("litellm_proxy.auth_cache_invalidation")[0][1] >= len(proxies):
+                        return
+            except httpx.TransportError:
+                pass
+            if time.monotonic() >= deadline:
+                raise SystemExit("Integration services or auth-cache subscribers did not become ready")
+            time.sleep(0.2)
+
+
+if __name__ == "__main__":
+    main()

.github/CODEOWNERS

@@ -4,7 +4,7 @@
 /ui/nginx.conf
 /ui/litellm-dashboard/src/lib/http/schema.d.ts
 /ui/litellm-dashboard/tsconfig.tsbuildinfo
-/model_prices_and_context_window.json @mateo-berri
-/litellm/model_prices_and_context_window_backup.json @mateo-berri
+/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
+/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
 /litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri
 /.github/CODEOWNERS @yuneng-berri

.github/ISSUE_TEMPLATE/config.yml

@@ -1,4 +1,4 @@
-blank_issues_enabled: true
+blank_issues_enabled: false
 contact_links:
   - name: Schedule Demo
     url: https://enterprise.litellm.ai/demo

.github/ci-coverage-allowlist.yml

@@ -106,11 +106,6 @@ dockerfiles:
       and lint workflows already exercise that output, so building the image adds no signal about it
     paths:
       - ui/Dockerfile
-  - reason: >-
-      The Rust gateway ships as its own chart and package with a separate release pipeline, so its
-      image is not part of this repo's Python image set
-    paths:
-      - litellm-rust/crates/ai-gateway/Dockerfile
   - reason: >-
       An example image under cookbook/ that is documentation rather than a shipped artifact
     paths:

.github/codeql/codeql-config.yml

@@ -14,6 +14,17 @@ query-filters:
       id: py/clear-text-logging-sensitive-data  # CWE-312
   - exclude:
       id: py/polynomial-redos                   # CWE-730
+  # Import resolution confuses stdlib types with management_endpoints/types.py.
+  # The generic cycle query also reports intentional deferred imports.
+  - exclude:
+      id: py/cyclic-import
+  - exclude:
+      id: py/unsafe-cyclic-import
+  # Known false positives on live settings and Protocol placeholders.
+  - exclude:
+      id: py/unused-global-variable
+  - exclude:
+      id: py/ineffectual-statement
 
 paths-ignore:
   - tests

.github/e2e-stack/assert_tests_ran.py

@@ -3,6 +3,9 @@ import xml.etree.ElementTree as ET
 from pathlib import Path
 from typing import Final
 
+sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests/e2e"))
+from coverage_registry.management_cases import MANAGEMENT_CASES
+
 
 def main() -> int:
     selected: Final = tuple(sys.argv[2:])
@@ -16,6 +19,17 @@ def main() -> int:
         case.get("file") for case in cases if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
     )
     missing: Final = tuple(path for path in selected if path not in passed)
+    required_nodes: Final = frozenset(case.node for case in MANAGEMENT_CASES if case.node.split("::", 1)[0] in selected)
+    passed_nodes: Final = frozenset(
+        prop.get("value")
+        for case in cases
+        if all(case.find(tag) is None for tag in ("skipped", "failure", "error"))
+        for prop in case.findall("./properties/property")
+        if prop.get("name") == "management_node"
+    )
+    missing_nodes: Final = required_nodes - passed_nodes
+    for node in sorted(missing_nodes):
+        _ = sys.stdout.write(f"::error::required management case did not pass: {node}\n")
     for path in selected:
         collected: Final = sum(case.get("file") == path for case in cases)
         skipped: Final = sum(case.get("file") == path and case.find("skipped") is not None for case in cases)
@@ -27,6 +41,7 @@ def main() -> int:
     if (
         selected
         and not missing
+        and not missing_nodes
         and not any(case.find(tag) is not None for case in cases for tag in ("failure", "error"))
     ):
         return 0

.github/e2e-stack/down.sh

@@ -10,7 +10,7 @@ for pid_file in "${STACK_DIR}"/pids/*.pid; do
   rm -f "${pid_file}"
 done
 
-for container in e2e-nginx e2e-valkey e2e-jaeger e2e-postgres; do
+for container in e2e-nginx e2e-keycloak e2e-valkey e2e-jaeger e2e-postgres; do
   docker rm -f "${container}" >/dev/null 2>&1
 done
 

.github/e2e-stack/oidc-profile.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "${REPO_ROOT}"
+exec uv run --no-sync python tests/e2e/idp.py "$@"

.github/e2e-stack/select_tests.py

@@ -11,6 +11,9 @@ UNSUPPORTED: Final = re.compile(
 )
 HARNESS: Final = re.compile(
     r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
+    r"|^tests/e2e/idp_realm\.json$"
+    r"|^tests/e2e/management/(management_client|jwt_actors|conftest)\.py$"
+    r"|^tests/e2e/coverage_registry/management_cases\.py$"
     r"|^tests/e2e/gateway/"
     r"|^\.github/e2e-stack/"
     r"|^\.github/workflows/test-e2e-changed\.yml$"

.github/e2e-stack/start-idp.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+KEYCLOAK_IMAGE="${E2E_KEYCLOAK_IMAGE:-quay.io/keycloak/keycloak@sha256:ff4257d0d64efbe99ed1ddfaf07765cc3c36dc7518bf8324d41961327f441c54}"
+KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}"
+POSTGRES_IMAGE="${E2E_POSTGRES_IMAGE:-postgres:16.6}"
+: "${DATABASE_HOST:?}" "${DATABASE_PORT:?}" "${DATABASE_USER:?}" "${DATABASE_PASSWORD:?}" "${DATABASE_NAME:?}"
+
+DB_HOST="${DATABASE_HOST}"
+DB_NETWORK_ARGS=(--network bridge)
+IDP_NETWORK_ARGS=(-p "127.0.0.1:${KEYCLOAK_PORT}:${KEYCLOAK_PORT}")
+if [[ "$(uname)" == "Linux" ]]; then
+  DB_NETWORK_ARGS=(--network host)
+  IDP_NETWORK_ARGS=(--network host)
+elif [[ "${DB_HOST}" == "127.0.0.1" || "${DB_HOST}" == "localhost" ]]; then
+  DB_HOST=host.docker.internal
+fi
+
+docker run --rm "${DB_NETWORK_ARGS[@]}" -e "PGPASSWORD=${DATABASE_PASSWORD}" \
+  "${POSTGRES_IMAGE}" psql -h "${DB_HOST}" -p "${DATABASE_PORT}" \
+  -U "${DATABASE_USER}" -d "${DATABASE_NAME}" -v ON_ERROR_STOP=1 \
+  -c 'CREATE SCHEMA IF NOT EXISTS keycloak' >/dev/null
+
+docker rm -f e2e-keycloak >/dev/null 2>&1 || true
+docker run -d --name e2e-keycloak "${IDP_NETWORK_ARGS[@]}" --memory 1536m \
+  -v "${REPO_ROOT}/tests/e2e/idp_realm.json:/opt/keycloak/data/import/realm.json:ro" \
+  -e KC_DB=postgres -e "KC_DB_URL_HOST=${DB_HOST}" -e "KC_DB_URL_PORT=${DATABASE_PORT}" \
+  -e "KC_DB_URL_DATABASE=${DATABASE_NAME}" -e KC_DB_SCHEMA=keycloak \
+  -e "KC_DB_USERNAME=${DATABASE_USER}" -e "KC_DB_PASSWORD=${DATABASE_PASSWORD}" \
+  -e KC_DB_POOL_INITIAL_SIZE=2 -e KC_DB_POOL_MIN_SIZE=2 -e KC_DB_POOL_MAX_SIZE=10 \
+  -e "KC_HTTP_PORT=${KEYCLOAK_PORT}" -e KC_BOOTSTRAP_ADMIN_USERNAME=admin \
+  -e KC_BOOTSTRAP_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret \
+  "${KEYCLOAK_IMAGE}" start-dev --import-realm >/dev/null
+
+deadline=$((SECONDS + ${E2E_KEYCLOAK_STARTUP_TIMEOUT:-300}))
+until curl -fsS --connect-timeout 2 --max-time 3 \
+  "http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/.well-known/openid-configuration" >/dev/null 2>&1; do
+  if ((SECONDS >= deadline)); then
+    echo 'e2e-stack: timed out waiting for the Keycloak realm' >&2
+    exit 1
+  fi
+  sleep 2
+done
+echo 'e2e-stack: Keycloak realm is up'

.github/e2e-stack/up.sh

@@ -25,6 +25,7 @@ DATABASE_PASSWORD="${E2E_DATABASE_PASSWORD:-dbpassword9090}"
 DATABASE_NAME="${E2E_DATABASE_NAME:-litellm}"
 JAEGER_OTLP_PORT="${E2E_JAEGER_OTLP_PORT:-4318}"
 JAEGER_QUERY_PORT="${E2E_JAEGER_QUERY_PORT:-16686}"
+KEYCLOAK_PORT="${E2E_KEYCLOAK_PORT:-8081}"
 
 MASTER_KEY="${LITELLM_MASTER_KEY:-sk-e2e-$(openssl rand -hex 16)}"
 
@@ -124,6 +125,9 @@ SERVER_ENV=(
   "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:${JAEGER_OTLP_PORT}"
   "SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem"
   "PYTHONPATH=${REPO_ROOT}"
+  "JWT_PUBLIC_KEY_URL=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e/protocol/openid-connect/certs"
+  "JWT_ISSUER=http://127.0.0.1:${KEYCLOAK_PORT}/realms/litellm-e2e"
+  "JWT_AUDIENCE=litellm-e2e"
 )
 if [[ -n "${VERTEXAI_CREDENTIALS:-}" ]]; then
   printf '%s' "${VERTEXAI_CREDENTIALS}" > "${STACK_DIR}/vertex-adc.json"
@@ -132,6 +136,8 @@ fi
 
 cd "${REPO_ROOT}"
 
+env "${SERVER_ENV[@]}" "E2E_KEYCLOAK_PORT=${KEYCLOAK_PORT}" bash .github/e2e-stack/start-idp.sh
+
 log "running migrations"
 env "${SERVER_ENV[@]}" uv run --no-sync python migrations/run.py >"${LOGS_DIR}/migrations.log" 2>&1
 
@@ -200,6 +206,9 @@ LITELLM_MASTER_KEY=${MASTER_KEY}
 REDIS_HOST=127.0.0.1
 REDIS_PORT=${REDIS_PORT}
 E2E_OTEL_QUERY_URL=http://127.0.0.1:${JAEGER_QUERY_PORT}
+E2E_KEYCLOAK_URL=http://127.0.0.1:${KEYCLOAK_PORT}
+E2E_KEYCLOAK_ADMIN_USER=admin
+E2E_KEYCLOAK_ADMIN_PASSWORD=e2e-ephemeral-idp-not-a-secret
 SSL_CERT_FILE=${CERTS_DIR}/ca-bundle.pem
 DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}
 EOF

.github/pull_request_template.md

@@ -47,6 +47,10 @@ After: the same request comes back with real token counts, so the dashboard show
 
 <!-- e.g., "Fixes #000" -->
 
+## Affected release
+
+<!-- Only for a fix to a regression in a released or rc version (perf, memory, crash, or behavior): name the version it regressed in, e.g. "regression in v1.100.0" or "since v1.101.0-rc.1", and add the `backport-stable` label so the fix is cherry-picked onto the rc line before the stable is tagged. Leave the section blank otherwise -->
+
 ## Linear ticket
 
 <!-- if you are an internal contributor, add "Resolves " followed by the Linear ticket e.g., "Resolves LIT-1234" to link the Linear ticket to the GitHub PR. If you don't have one, leave the section blank rather than guessing -->
@@ -97,7 +101,8 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
      For bug fixes: Before shows the reproduction, After shows the same steps passing
      For new features: Before shows the capability missing, After shows it working end-to-end
      If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), make each endpoint its own case, not just one
-     For UI changes: before/after screenshots under the same headings -->
+     For UI changes: before/after screenshots under the same headings
+     If the main use case runs through a coding tool like Claude Code or Codex, drive that tool interactively the way the user does (never `claude -p`, `codex exec`, or curl on its own) and embed before/after screenshots of its pane under the same headings; curl replays and headless runs can follow as extra cases, never as the only proof -->
 
 ## Type
 
@@ -127,6 +132,8 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
      - Low: anything else worth noting: naming, cleanup, an edge case nobody hits
      Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a
      human reader
+     If you assumed something instead of testing it, e.g. "only reproduces with X on" or "no
+     user-observable behavior difference", list it here too with what breaks if it is wrong
      Leave this section empty if there are none -->
 
 ## QA runbook
@@ -152,3 +159,4 @@ Example checklists:
 ## Final Attestation
 
 - [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR
+

.github/scripts/assert_ci_coverage.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import ast
+import json
 import operator
 import pathlib
 import re
@@ -498,6 +499,104 @@ def _check_shards() -> int:
     return 0
 
 
+def _integration_ownership(repo_root: pathlib.Path = REPO_ROOT) -> tuple[frozenset[str], tuple[Finding, ...]]:
+    manifest: Final = repo_root / "tests/integration/contracts.json"
+    if not manifest.exists():
+        return frozenset(), ()
+    entries: Final = json.loads(manifest.read_text())
+    paths: Final = frozenset(node.split("::", 1)[0] for node in entries["tests"])
+    browser_paths: Final = frozenset(node.split("::", 1)[0] for node in entries.get("browser", {}))
+    circle_path: Final = repo_root / ".circleci/config.yml"
+    circle: Final = yaml.safe_load(circle_path.read_text()) if circle_path.exists() else {}
+    steps: Final = circle.get("jobs", {}).get("integration_contracts", {}).get("steps", ())
+    invoked: Final = any(
+        ".circleci/scripts/run_integration.sh" in scalar.value
+        for scalar in _scalars(steps, "integration_contracts")
+        if scalar.key == "command"
+    )
+    scheduled: Final = frozenset(
+        suite
+        for job in circle.get("workflows", {}).get("integration", {}).get("jobs", ())
+        if isinstance(job, dict) and "integration_contracts" in job
+        for suite in job["integration_contracts"]
+        .get("matrix", {})
+        .get("parameters", {})
+        .get("suite", (job["integration_contracts"].get("suite"),))
+        if isinstance(suite, str)
+    )
+    required: Final = (frozenset({"browser"}) if browser_paths else frozenset()) | frozenset(
+        group
+        for group, folders in entries["groups"].items()
+        if any(any(path.startswith(f"tests/integration/{folder}/") for folder in folders) for path in paths)
+    )
+    ungrouped: Final = frozenset(
+        path
+        for path in paths
+        if sum(
+            any(path.startswith(f"tests/integration/{folder}/") for folder in folders)
+            for folders in entries["groups"].values()
+        )
+        != 1
+    )
+    gha_tokens: Final = _invoked_test_tokens(
+        scalar
+        for path in (repo_root / ".github/workflows").glob("*.y*ml")
+        for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
+    )
+    findings: Final = tuple(
+        Finding(path, "integration contract is also selected by GitHub Actions")
+        for path in paths
+        if any(_token_covers(token, path) for token in gha_tokens)
+    ) + tuple(
+        Finding(path, "canonical integration test file is missing")
+        for path in paths
+        if not (repo_root / path).is_file()
+    )
+    browser_commands: Final = tuple(
+        scalar.value
+        for path in (repo_root / ".github/workflows").glob("*.y*ml")
+        for scalar in _scalars(yaml.safe_load(path.read_text()), path.name)
+        if scalar.key in {"run", "command"}
+    )
+    browser_findings: Final = tuple(
+        Finding(path, "browser integration contract is explicitly selected by GitHub Actions")
+        for path in browser_paths
+        if any(
+            path in command
+            or pathlib.Path(path).name in command
+            or "integrationCritical" in command
+            or "integration.config.ts" in command
+            or ("run_integration.sh" in command and "browser" in command)
+            for command in browser_commands
+        )
+    ) + tuple(
+        Finding(path, "canonical browser integration file is missing")
+        for path in browser_paths
+        if not (repo_root / path).is_file()
+    )
+    default_browser: Final = repo_root / "tests/e2e/ui/playwright.config.ts"
+    exclusion_findings: Final = (
+        (
+            Finding(
+                str(default_browser.relative_to(repo_root)),
+                "default Playwright selection must exclude integrationCritical",
+            ),
+        )
+        if browser_paths
+        and (not default_browser.exists() or "**/integrationCritical/**" not in default_browser.read_text())
+        else ()
+    )
+    group_findings: Final = tuple(
+        Finding(group, "canonical integration group is not scheduled by CircleCI")
+        for group in sorted(required - scheduled)
+    ) + tuple(Finding(path, "canonical node must have exactly one integration group") for path in sorted(ungrouped))
+    if not paths or not invoked or not scheduled:
+        return frozenset(), findings + (
+            Finding(str(manifest.relative_to(repo_root)), "dedicated CircleCI runner is missing"),
+        )
+    return paths | browser_paths, findings + group_findings + browser_findings + exclusion_findings
+
+
 def main() -> int:
     if "--shards" in sys.argv[1:]:
         return _check_shards()
@@ -507,7 +606,8 @@ def main() -> int:
     allowlist = _load_allowlist()
     scalars = _all_scalars()
 
-    test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars))
+    integration_paths, ownership_findings = _integration_ownership()
+    test_findings = _uncovered_tests(allowlist, _invoked_test_tokens(scalars) | integration_paths) + ownership_findings
     dockerfile_findings = _uncovered_dockerfiles(allowlist, _built_dockerfile_tokens(scalars))
     stale_findings = _stale_allowlist_paths(allowlist, test_files=_test_files(), dockerfiles=_dockerfiles())
 

.github/scripts/auto_merge_price_sync.py

@@ -0,0 +1,393 @@
+"""Auto-merge the provider-info-sync bot's cost-map pull requests.
+
+Evaluates every gate (author allowlist, cost-map-only diff, required and
+non-required checks, human reviews) and merges with a merge commit when
+all of them hold. Every hold reason is logged; the process exits 0 on hold
+and 1 only on API or programming errors.
+``DRY_RUN=1`` prints the verdict without calling the merge endpoint.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Final
+
+REPO_ROOT: Final = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+CLASSIFY_SCRIPT: Final = os.path.join(REPO_ROOT, ".circleci", "scripts", "classify_changes.sh")
+API_ROOT: Final = "https://api.github.com"
+CHANGED_FILE_CEILING: Final = 3000
+OK_CHECK_CONCLUSIONS: Final = frozenset({"success", "skipped", "neutral"})
+
+
+@dataclass(frozen=True, slots=True)
+class PullRequest:
+    number: int
+    title: str
+    author_login: str
+    state: str
+    draft: bool
+    mergeable: bool | None
+    mergeable_state: str
+    head_sha: str
+
+
+@dataclass(frozen=True, slots=True)
+class CheckRun:
+    name: str
+    status: str
+    conclusion: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class CommitStatus:
+    context: str
+    state: str
+
+
+@dataclass(frozen=True, slots=True)
+class Review:
+    author_login: str
+    state: str
+    body: str
+    commit_id: str
+    submitted_at: datetime
+
+
+@dataclass(frozen=True, slots=True)
+class Verdict:
+    merge: bool
+    reasons: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class EvaluationInputs:
+    pr: PullRequest
+    changed_files: tuple[str, ...]
+    required_contexts: frozenset[str]
+    check_runs: tuple[CheckRun, ...]
+    statuses: tuple[CommitStatus, ...]
+    reviews: tuple[Review, ...]
+    self_check_name: str
+    author_allowlist: frozenset[str]
+
+
+def _is_bot_login(login: str) -> bool:
+    return login.lower().endswith("[bot]")
+
+
+def _classify(changed_files: Sequence[str]) -> str:
+    result: Final = subprocess.run(
+        ["bash", CLASSIFY_SCRIPT, "cost-map-only"],
+        input="\n".join(changed_files),
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if result.returncode != 0:
+        return "error"
+    return result.stdout.strip()
+
+
+def evaluate(
+    inputs: EvaluationInputs,
+    *,
+    classify: Callable[[Sequence[str]], str] = _classify,
+) -> Verdict:
+    pr: Final = inputs.pr
+    reasons: list[str] = []
+
+    if pr.author_login.lower() not in {login.lower() for login in inputs.author_allowlist}:
+        reasons.append(f"author {pr.author_login!r} not in allowlist")
+    if pr.state != "open":
+        reasons.append("pr not open")
+    if pr.draft:
+        reasons.append("pr is a draft")
+    if pr.mergeable is None:
+        reasons.append("mergeability unknown")
+    elif not pr.mergeable:
+        reasons.append("pr not mergeable")
+    if pr.mergeable_state == "dirty":
+        reasons.append("pr has merge conflicts")
+
+    if len(inputs.changed_files) > CHANGED_FILE_CEILING:
+        reasons.append(f"changed file count {len(inputs.changed_files)} over {CHANGED_FILE_CEILING} ceiling")
+    else:
+        decision: Final = classify(inputs.changed_files)
+        if decision != "run":
+            reasons.append("changed files outside the cost-map-only set")
+
+    green_runs: Final = frozenset(run.name for run in inputs.check_runs if run.conclusion in OK_CHECK_CONCLUSIONS)
+    green_statuses: Final = frozenset(status.context for status in inputs.statuses if status.state == "success")
+    for context in sorted(inputs.required_contexts):
+        if context not in green_runs and context not in green_statuses:
+            reasons.append(f"required check {context!r} not green")
+    for run in inputs.check_runs:
+        if run.name == inputs.self_check_name:
+            continue
+        if run.status != "completed" or run.conclusion not in OK_CHECK_CONCLUSIONS:
+            reasons.append(f"check run {run.name!r} is {run.status}/{run.conclusion}")
+    for status in inputs.statuses:
+        if status.state != "success":
+            reasons.append(f"commit status {status.context!r} is {status.state}")
+
+    latest_state_by_reviewer: Final[dict[str, str]] = {}
+    for review in sorted(inputs.reviews, key=lambda review: review.submitted_at):
+        if _is_bot_login(review.author_login):
+            continue
+        latest_state_by_reviewer[review.author_login] = review.state
+    for reviewer, state in latest_state_by_reviewer.items():
+        if state == "CHANGES_REQUESTED":
+            reasons.append(f"changes requested by {reviewer}")
+
+    return Verdict(merge=not reasons, reasons=tuple(reasons))
+
+
+def _request(token: str, method: str, path: str, body: Mapping[str, object] | None = None) -> object:
+    url: Final = path if path.startswith("http") else f"{API_ROOT}{path}"
+    data: Final = None if body is None else json.dumps(body).encode("utf-8")
+    request: Final = urllib.request.Request(
+        url,
+        data=data,
+        method=method,
+        headers={
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+    )
+    with urllib.request.urlopen(request) as response:
+        return json.loads(response.read().decode("utf-8"))
+
+
+def _request_allow_fail(
+    token: str, method: str, path: str, body: Mapping[str, object] | None = None
+) -> tuple[int, object | None]:
+    url: Final = path if path.startswith("http") else f"{API_ROOT}{path}"
+    data: Final = None if body is None else json.dumps(body).encode("utf-8")
+    request: Final = urllib.request.Request(
+        url,
+        data=data,
+        method=method,
+        headers={
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+    )
+    try:
+        with urllib.request.urlopen(request) as response:
+            return response.status, json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as exc:
+        return exc.code, None
+
+
+def _items(payload: object, key: str | None = None) -> tuple[object, ...]:
+    source: Final = payload.get(key) if key and isinstance(payload, Mapping) else payload
+    if not isinstance(source, list):
+        return ()
+    return tuple(source)
+
+
+def _paginate(token: str, path: str, key: str | None = None) -> list[object]:
+    separator: Final = "&" if "?" in path else "?"
+    results: list[object] = []
+    for page in range(1, 10_000):
+        batch: Final = _items(_request(token, "GET", f"{path}{separator}per_page=100&page={page}"), key)
+        results.extend(batch)
+        if len(batch) < 100:
+            return results
+    return results
+
+
+def _text(value: object) -> str:
+    return value if isinstance(value, str) else ""
+
+
+def _int(value: object) -> int:
+    return value if isinstance(value, int) else 0
+
+
+def _bool(value: object) -> bool:
+    return value is True
+
+
+def _nested(value: object, *keys: str) -> object:
+    current: object = value
+    for key in keys:
+        if not isinstance(current, Mapping):
+            return None
+        current = current.get(key)
+    return current
+
+
+def _parse_time(value: object) -> datetime:
+    text: Final = _text(value)
+    if not text:
+        return datetime.min.replace(tzinfo=timezone.utc)
+    return datetime.fromisoformat(text.replace("Z", "+00:00"))
+
+
+def _load_pr(token: str, repo: str, number: int) -> PullRequest:
+    data: Final = _request(token, "GET", f"/repos/{repo}/pulls/{number}")
+    if not isinstance(data, Mapping):
+        raise RuntimeError(f"unexpected pull payload for #{number}")
+    return PullRequest(
+        number=number,
+        title=_text(data.get("title")),
+        author_login=_text(_nested(data, "user", "login")),
+        state=_text(data.get("state")),
+        draft=_bool(data.get("draft")),
+        mergeable=data.get("mergeable") if isinstance(data.get("mergeable"), bool) else None,
+        mergeable_state=_text(data.get("mergeable_state")),
+        head_sha=_text(_nested(data, "head", "sha")),
+    )
+
+
+def _list_candidate_prs(token: str, repo: str, base: str, allowlist: frozenset[str]) -> list[int]:
+    candidates: Final = _paginate(token, f"/repos/{repo}/pulls?state=open&base={base}")
+    return [
+        _int(item.get("number"))
+        for item in candidates
+        if isinstance(item, Mapping) and _text(_nested(item, "user", "login")).lower() in allowlist
+    ]
+
+
+def _changed_files(token: str, repo: str, number: int) -> tuple[str, ...]:
+    files: Final = _paginate(token, f"/repos/{repo}/pulls/{number}/files")
+    return tuple(_text(item.get("filename")) for item in files if isinstance(item, Mapping))
+
+
+def _required_contexts(token: str, repo: str, base: str) -> frozenset[str]:
+    payload: Final = _request(token, "GET", f"/repos/{repo}/rules/branches/{base}")
+    contexts: set[str] = set()
+    for rule in _items(payload):
+        if not isinstance(rule, Mapping) or rule.get("type") != "required_status_checks":
+            continue
+        checks: Final = _nested(rule, "parameters", "required_status_checks")
+        for check in _items(checks):
+            if isinstance(check, Mapping):
+                context: Final = _text(check.get("context"))
+                if context:
+                    contexts.add(context)
+    return frozenset(contexts)
+
+
+def _check_runs(token: str, repo: str, sha: str) -> tuple[CheckRun, ...]:
+    runs: Final = _paginate(token, f"/repos/{repo}/commits/{sha}/check-runs", key="check_runs")
+    return tuple(
+        CheckRun(
+            name=_text(item.get("name")),
+            status=_text(item.get("status")),
+            conclusion=item.get("conclusion") if isinstance(item.get("conclusion"), str) else None,
+        )
+        for item in runs
+        if isinstance(item, Mapping)
+    )
+
+
+def _statuses(token: str, repo: str, sha: str) -> tuple[CommitStatus, ...]:
+    payload: Final = _request(token, "GET", f"/repos/{repo}/commits/{sha}/status")
+    return tuple(
+        CommitStatus(context=_text(item.get("context")), state=_text(item.get("state")))
+        for item in _items(payload, "statuses")
+        if isinstance(item, Mapping)
+    )
+
+
+def _reviews(token: str, repo: str, number: int) -> tuple[Review, ...]:
+    reviews: Final = _paginate(token, f"/repos/{repo}/pulls/{number}/reviews")
+    return tuple(
+        Review(
+            author_login=_text(_nested(item, "user", "login")),
+            state=_text(item.get("state")),
+            body=_text(item.get("body")),
+            commit_id=_text(item.get("commit_id")),
+            submitted_at=_parse_time(item.get("submitted_at")),
+        )
+        for item in reviews
+        if isinstance(item, Mapping)
+    )
+
+
+def _mergeable_or_refetch(token: str, repo: str, pr: PullRequest) -> PullRequest:
+    if pr.mergeable is not None:
+        return pr
+    time.sleep(5)
+    return _load_pr(token, repo, pr.number)
+
+
+def _gather_inputs(
+    token: str,
+    repo: str,
+    number: int,
+    base: str,
+    self_check_name: str,
+    allowlist: frozenset[str],
+) -> EvaluationInputs:
+    pr: Final = _mergeable_or_refetch(token, repo, _load_pr(token, repo, number))
+    return EvaluationInputs(
+        pr=pr,
+        changed_files=_changed_files(token, repo, number),
+        required_contexts=_required_contexts(token, repo, base),
+        check_runs=_check_runs(token, repo, pr.head_sha),
+        statuses=_statuses(token, repo, pr.head_sha),
+        reviews=_reviews(token, repo, number),
+        self_check_name=self_check_name,
+        author_allowlist=allowlist,
+    )
+
+
+def merge_request_body(pr: PullRequest) -> dict[str, str]:
+    return {"merge_method": "merge", "commit_title": f"{pr.title} (#{pr.number})", "sha": pr.head_sha}
+
+
+def _merge(token: str, repo: str, pr: PullRequest) -> None:
+    status, _ = _request_allow_fail(token, "PUT", f"/repos/{repo}/pulls/{pr.number}/merge", merge_request_body(pr))
+    if status in (200, 405, 409):
+        print(f"auto-merge-price-sync: PR #{pr.number} merge call returned {status}")
+        return
+    raise RuntimeError(f"merge call for PR #{pr.number} returned {status}")
+
+
+def main() -> int:
+    token: Final = os.environ.get("GH_TOKEN", "")
+    repo: Final = os.environ.get("REPO", "")
+    base: Final = os.environ.get("BASE_BRANCH", "main")
+    dry_run: Final = os.environ.get("DRY_RUN", "") != ""
+    self_check_name: Final = os.environ.get("SELF_CHECK_NAME", "auto-merge-price-sync")
+    allowlist: Final = frozenset(login.lower() for login in os.environ.get("PR_AUTHOR_ALLOWLIST", "").split() if login)
+    if not token:
+        print("auto-merge-price-sync: app credentials not configured")
+        return 0
+    if not repo:
+        print("auto-merge-price-sync: REPO not set", file=sys.stderr)
+        return 1
+
+    pr_number_env: Final = os.environ.get("PR_NUMBER", "")
+    candidates: Final = [int(pr_number_env)] if pr_number_env else _list_candidate_prs(token, repo, base, allowlist)
+    for number in candidates:
+        inputs: Final = _gather_inputs(token, repo, number, base, self_check_name, allowlist)
+        verdict: Final = evaluate(inputs)
+        for reason in verdict.reasons:
+            print(f"auto-merge-price-sync: PR #{number} hold: {reason}")
+        if not verdict.merge:
+            continue
+        print(f"auto-merge-price-sync: PR #{number} all gates green")
+        if dry_run:
+            print(f"auto-merge-price-sync: DRY_RUN merge suppressed for PR #{number}")
+            continue
+        _merge(token, repo, inputs.pr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/auto_update_price_and_context_window_file.py

@@ -1,6 +1,8 @@
 import asyncio
 import aiohttp
 import json
+import math
+from typing import Any
 
 # Asynchronously fetch data from a given URL
 async def fetch_data(url):
@@ -21,11 +23,157 @@ async def fetch_data(url):
         print("Error fetching data from URL:", e)
         return None
 
+
+FRIENDLI_API_URL = "https://api.friendli.ai/serverless/v1/models"
+FRIENDLI_PROVIDER = "friendliai"
+
+INHERITABLE_BASE_KEYS = (
+    "supports_pdf_input",
+    "supports_assistant_prefill",
+    "supports_adaptive_thinking",
+    "supports_output_config",
+)
+
+REASONING_EFFORT_LEVEL_ORDER = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
+
+
+def _find_base_model_entry(base_model: str, local_data: dict) -> str | None:
+    if not base_model:
+        return None
+    bm_tail = base_model.split("/")[-1].lower()
+    if base_model in local_data:
+        return base_model
+    for key in local_data:
+        if key.startswith("sample_spec") or key == "fallback_generalizations":
+            continue
+        if key.split("/")[-1].lower() == bm_tail:
+            return key
+    return None
+
+
+def _reasoning_effort_levels(reasoning_options: list) -> list:
+    offered = {
+        val
+        for opt in reasoning_options or []
+        if opt.get("type") == "effort"
+        for val in opt.get("values", [])
+    }
+    return [level for level in REASONING_EFFORT_LEVEL_ORDER if level in offered]
+
+
+def _valid_token_price(value: object) -> bool:
+    try:
+        price = float(value)  # pyright: ignore[reportArgumentType]  # non-numeric values are rejected via the except
+    except (TypeError, ValueError):
+        return False
+    return math.isfinite(price) and price >= 0
+
+
+def _has_valid_token_prices(pricing: dict | None) -> bool:
+    prices = pricing or {}
+    return _valid_token_price(prices.get("input")) and _valid_token_price(prices.get("output"))
+
+
+def _pricing(pricing: dict) -> dict:
+    out: dict[str, Any] = {}
+    if not pricing:
+        return out
+    if "input" in pricing:
+        out["input_cost_per_token"] = float(pricing["input"])
+    if "output" in pricing:
+        out["output_cost_per_token"] = float(pricing["output"])
+    if "input_cache_read" in pricing and pricing["input_cache_read"] is not None:
+        out["cache_read_input_token_cost"] = float(pricing["input_cache_read"])
+    return out
+
+
+def _modality_flags(input_mods: list) -> dict:
+    mods = input_mods or []
+    has_image = "image" in mods
+    return {
+        "supports_vision": has_image,
+        "supports_image_input": has_image,
+        "supports_video_input": "video" in mods,
+    }
+
+
+def transform_friendli_data(data: list, local_data: dict) -> dict:
+    transformed: dict[str, dict] = {}
+    if not data:
+        return transformed
+    for model in data:
+        # An unpriced row must never wholesale-replace an already priced local entry:
+        # missing prices cost-calculate as zero, silently zeroing tracked spend
+        if not _has_valid_token_prices(model.get("pricing")):
+            continue
+        model_id = model["id"]
+        base_model = model.get("base_model") or ""
+        entry: dict[str, Any] = {
+            "litellm_provider": FRIENDLI_PROVIDER,
+        }
+
+        base_key = _find_base_model_entry(base_model, local_data)
+        if base_key:
+            base_entry = local_data[base_key]
+            for k in INHERITABLE_BASE_KEYS:
+                if k in base_entry:
+                    entry[k] = base_entry[k]
+
+        ctx = model.get("context_length")
+        if ctx is not None:
+            entry["max_input_tokens"] = int(ctx)
+        max_out = model.get("max_completion_tokens")
+        if max_out is not None:
+            entry["max_output_tokens"] = int(max_out)
+            entry["max_tokens"] = int(max_out)
+
+        pricing = _pricing(model.get("pricing", {}))
+        entry.update(pricing)
+        entry["supports_prompt_caching"] = "cache_read_input_token_cost" in pricing
+
+        reasoning = model.get("reasoning") is True
+        entry["supports_reasoning"] = reasoning
+        if reasoning:
+            entry["reasoning_effort_levels"] = _reasoning_effort_levels(
+                model.get("reasoning_options", [])
+            )
+
+        func = model.get("functionality", {})
+        entry["supports_function_calling"] = func.get("tool_call") is True
+        entry["supports_parallel_function_calling"] = func.get("parallel_tool_call") is True
+        is_struct = func.get("structured_output") is True
+        entry["supports_response_schema"] = is_struct
+        entry["supports_native_structured_output"] = is_struct
+        entry["supports_system_messages"] = func.get("system_messages") is True
+        entry["supports_tool_choice"] = func.get("tool_choice") is True
+
+        entry.update(_modality_flags(model.get("input_modalities", [])))
+
+        entry["mode"] = model.get("mode", "chat")
+
+        desc = model.get("description")
+        if desc:
+            entry["comment"] = desc
+
+        dep = model.get("deprecation_date")
+        if dep:
+            entry["deprecation_date"] = dep.split("T")[0]
+
+        entry["source"] = FRIENDLI_API_URL
+
+        transformed[f"{FRIENDLI_PROVIDER}/{model_id}"] = entry
+    return transformed
+
 # Synchronize local data with remote data
-def sync_local_data_with_remote(local_data, remote_data):
+def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset()):
     # Update existing keys in local_data with values from remote_data
+    # (replace_keys entries are swapped wholesale so a field the remote catalog
+    #  dropped, e.g. cache pricing, cannot survive as a stale value)
     for key in (set(local_data) & set(remote_data)):
-        local_data[key].update(remote_data[key])
+        if key in replace_keys:
+            local_data[key] = remote_data[key]
+        else:
+            local_data[key].update(remote_data[key])
 
     # Add new keys from remote_data to local_data
     for key in (set(remote_data) - set(local_data)):
@@ -46,6 +194,8 @@ def write_to_file(file_path, data):
 # Update the existing models and add the missing models for OpenRouter
 def transform_openrouter_data(data):
     transformed = {}
+    if not data:
+        return transformed
     for row in data:
         # Add the fields 'max_tokens' and 'input_cost_per_token'
         obj = {
@@ -84,7 +234,14 @@ def transform_openrouter_data(data):
 # Update the existing models and add the missing models for Vercel AI Gateway
 def transform_vercel_ai_gateway_data(data):
     transformed = {}
+    if not data:
+        return transformed
     for row in data:
+        # Rows without token pricing or token limits (video/embedding models) previously KeyError'd the whole sync
+        if any(row.get(k) is None for k in ("context_window", "max_tokens")) or any(
+            row.get("pricing", {}).get(k) is None for k in ("input", "output")
+        ):
+            continue
         obj = {
             "max_tokens": row["context_window"],
             "input_cost_per_token": float(row["pricing"]["input"]),
@@ -143,13 +300,16 @@ def main():
     vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
     # Transform the fetched Vercel AI Gateway data
     vercel_data = transform_vercel_ai_gateway_data(vercel_data)
+
+    friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL))
+    friendli_data = transform_friendli_data(friendli_data, local_data)
     
     # Combine both datasets
-    all_remote_data = {**openrouter_data, **vercel_data}
+    all_remote_data = {**openrouter_data, **vercel_data, **friendli_data}
 
     # If both local and openrouter data are available, synchronize and save
     if local_data and all_remote_data:
-        sync_local_data_with_remote(local_data, all_remote_data)
+        sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data))
         write_to_file(local_file_path, local_data)
     else:
         print("Failed to fetch model data from either local file or URL.")

.github/scripts/verify_linux_native_wheel.py

@@ -205,7 +205,7 @@ def main(
     native_module: Final = load_native_module(native_path)
     native_module_loads: Final = native_module is not None
     panic_test_hook_absent: Final = native_module is not None and not hasattr(native_module, "_panic_for_test")
-    native_size_limit: Final = 20_000_000
+    native_size_limit: Final = 25_000_000
     native_size_within_limit: Final = native_member.file_size <= native_size_limit
     validations: Final = (
         (f"Python tag is {EXPECTED_PYTHON_TAG}", python_tag == EXPECTED_PYTHON_TAG),
@@ -222,7 +222,7 @@ def main(
         ("Python extension entry point is present", extension_entry_point_present),
         ("Native module loads", native_module_loads),
         ("Production module omits the panic test hook", panic_test_hook_absent),
-        ("Native extension does not exceed 20 MB", native_size_within_limit),
+        ("Native extension does not exceed 25 MB", native_size_within_limit),
         ("Wheel contents are valid", not unexpected_members),
     )
 

.github/workflows/_test-unit-base.yml

@@ -57,6 +57,7 @@ permissions:
 
 env:
   UV_PYTHON: "3.12"
+  LITELLM_LOCAL_MODEL_COST_MAP: "True"
 
 jobs:
   run:
@@ -113,6 +114,7 @@ jobs:
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 8
         run: |
+          diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
           uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
 

.github/workflows/auto-merge-price-sync.yml

@@ -0,0 +1,61 @@
+name: auto-merge-price-sync
+
+on:
+  issue_comment:
+    types: [created, edited]
+  check_suite:
+    types: [completed]
+  status: {}
+  schedule:
+    - cron: "*/30 * * * *"
+  workflow_dispatch:
+    inputs:
+      pr-number:
+        description: "Evaluate only this PR number (empty = scan all open sync-bot PRs)"
+        required: false
+        default: ""
+
+permissions:
+  contents: read
+  pull-requests: read
+  checks: read
+  statuses: read
+
+concurrency:
+  group: auto-merge-price-sync
+  cancel-in-progress: false
+
+jobs:
+  auto-merge-price-sync:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    env:
+      PROVIDER_INFO_SYNC_APP_ID: ${{ secrets.PROVIDER_INFO_SYNC_APP_ID }}
+      PROVIDER_INFO_SYNC_APP_PRIVATE_KEY: ${{ secrets.PROVIDER_INFO_SYNC_APP_PRIVATE_KEY }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Mint app token
+        id: app-token
+        if: ${{ env.PROVIDER_INFO_SYNC_APP_ID != '' && env.PROVIDER_INFO_SYNC_APP_PRIVATE_KEY != '' }}
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ secrets.PROVIDER_INFO_SYNC_APP_ID }}
+          private-key: ${{ secrets.PROVIDER_INFO_SYNC_APP_PRIVATE_KEY }}
+
+      - name: Auto-merge eligible sync PRs
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ (github.event.issue.pull_request && github.event.issue.number) || github.event.inputs.pr-number || '' }}
+          BASE_BRANCH: main
+          PR_AUTHOR_ALLOWLIST: "berriai-litellm-provider-info-sync[bot]"
+          SELF_CHECK_NAME: auto-merge-price-sync
+        run: python3 .github/scripts/auto_merge_price_sync.py

.github/workflows/guard-main-branch.yml

@@ -1,42 +0,0 @@
-name: Guard main branch
-
-on:
-  pull_request:
-    branches:
-      - main
-  merge_group:
-
-permissions: {}
-
-# DO NOT RENAME the job's `name:` — it is referenced by GitHub branch
-# protection as a required status check on `main`. Renaming silently
-# breaks the gate.
-jobs:
-  guard:
-    name: Verify PR source branch
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    steps:
-      - name: Reject merge_group events
-        if: github.event_name == 'merge_group'
-        run: |
-          echo "::error::Merge queue is not supported for main. Disable merge queue or update this guard."
-          exit 1
-      - name: Check head branch name
-        env:
-          HEAD_REF: ${{ github.head_ref }}
-          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
-          BASE_REPO: ${{ github.repository }}
-        run: |
-          echo "PR head repo: $HEAD_REPO"
-          echo "PR head branch: $HEAD_REF"
-          if [ "$HEAD_REPO" != "$BASE_REPO" ]; then
-            echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against 'litellm_internal_staging' instead."
-            exit 1
-          fi
-          if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then
-            echo "Allowed source branch."
-            exit 0
-          fi
-          echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_internal_staging' instead."
-          exit 1

.github/workflows/image-scan.yml

@@ -26,6 +26,7 @@ on:
       - ui/Dockerfile
       - ui/nginx.conf
       - .github/workflows/image-scan.yml
+      - .grype.yaml
   schedule:
     - cron: "41 6 * * *"
   workflow_dispatch:
@@ -93,6 +94,7 @@ jobs:
           GRYPE_MATCH_PYTHON_USING_CPES: "true"
         run: |
           "$RUNNER_TEMP/grype" litellm-image-scan:${{ github.sha }} \
+            --config .grype.yaml \
             --only-fixed \
             --fail-on high \
             --output table

.github/workflows/test-code-quality.yml

@@ -81,7 +81,7 @@ jobs:
         run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py
 
       - name: test_e2e_changed_gate
-        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py
+        run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py tests/code_coverage_tests/test_e2e_idp_stack.py
 
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
@@ -178,7 +178,7 @@ jobs:
           version: "0.10.9"
 
       - name: Install dependencies
-        run: uv sync --frozen --extra proxy --python 3.10
+        run: uv sync --frozen --extra proxy --extra cli --python 3.10
 
       - run: uv run --no-sync python --version
 
@@ -187,3 +187,6 @@ jobs:
 
       - name: Check litellm CLI
         run: uv run --no-sync litellm --version
+
+      - name: Check lite CLI
+        run: uv run --no-sync lite version

.github/workflows/test-e2e-changed.yml

@@ -27,6 +27,8 @@ jobs:
           sparse-checkout: |
             .github/e2e-stack
             tests/e2e/access_control
+            tests/e2e/management/test_jwt_management_e2e.py
+            tests/e2e/other/test_jwt_auth_e2e.py
           persist-credentials: false
           ref: ${{ github.sha }}
 
@@ -45,7 +47,8 @@ jobs:
             --jq '.[] | select(.status != "removed") | .filename')"
           gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha' | grep -Fxq "${HEAD_SHA}"
           tests="$(printf '%s\n' "${files}" \
-            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py)"
+            | python3 .github/e2e-stack/select_tests.py tests/e2e/access_control/test_*.py \
+                tests/e2e/management/test_jwt_management_e2e.py tests/e2e/other/test_jwt_auth_e2e.py)"
           echo "tests=${tests}" >> "${GITHUB_OUTPUT}"
           if [ -n "${tests}" ]; then
             echo "any=true" >> "${GITHUB_OUTPUT}"
@@ -180,7 +183,7 @@ jobs:
             log="${RUNNER_TEMP}/e2e-pass-${pass}.log"
             echo "::group::pass ${pass} of 3"
             set +e
-            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v -p no:cacheprovider \
+            uv run --no-sync pytest "${test_files[@]}" --rootdir=. -v --reruns 0 -p no:cacheprovider \
               -o junit_family=xunit1 --junitxml="${report}" > "${log}" 2>&1
             status=$?
             uv run --no-sync python .github/e2e-stack/assert_tests_ran.py "${report}" "${test_files[@]}"

.github/workflows/test-e2e-redis-chaos.yml

@@ -0,0 +1,103 @@
+name: "Redis Chaos E2E"
+
+on:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      ref:
+        description: "Commit SHA or ref to test. Defaults to the ref the workflow was triggered on"
+        required: false
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  redis-chaos-e2e:
+    runs-on: ubuntu-latest-16-cores
+    timeout-minutes: 30
+    services:
+      postgres:
+        image: postgres:16.6@sha256:557fea37a744d5f4c8faab304b0a90858b53ab119735a88c131fd19dab802f36
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+      valkey:
+        image: valkey/valkey:8.1.4@sha256:81db6d39e1bba3b3ff32bd3a1b19a6d69690f94a3954ec131277b9a26b95b3aa
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "valkey-cli ping"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-redis-chaos-e2e
+      LITELLM_LOG: WARNING
+      JSON_LOGS: "true"
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+          ref: ${{ inputs.ref || github.sha }}
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --group e2e-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Start a multi-worker proxy on the chaos config
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/redis_chaos_ci_config.yml --port 4000 --num_workers 4 > proxy.log 2>&1 &
+          echo "E2E_PROXY_PID=$!" >> "$GITHUB_ENV"
+          echo "E2E_PROXY_LOG=$(pwd)/proxy.log" >> "$GITHUB_ENV"
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Run the Redis chaos load test
+        env:
+          E2E_REDIS_CHAOS: "1"
+          LITELLM_PROXY_URL: http://localhost:4000
+          REDIS_HOST: 127.0.0.1
+          REDIS_PORT: "6379"
+        run: |
+          uv run --no-sync pytest tests/e2e/load/test_redis_chaos_e2e.py -v --tb=short -rA -s
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log

.github/workflows/test-linting.yml

@@ -113,7 +113,7 @@ jobs:
       - name: Check ruff format
         if: steps.changes.outputs.decision != 'skip'
         run: |
-          git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
+          git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- ':(glob)litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
           if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
             echo "No changed litellm Python files to check with ruff format."
             exit 0
@@ -172,7 +172,7 @@ jobs:
       - name: Check tests/e2e basedpyright (zero errors)
         if: steps.changes.outputs.decision != 'skip'
         run: |
-          if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
+          if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- ':(glob)tests/e2e/**/*.py' | grep -q .; then
             uv run --no-sync basedpyright tests/e2e
           else
             echo "No changed tests/e2e Python files; skipping."

.github/workflows/test-rust.yml

@@ -4,8 +4,22 @@ on:
   push:
     paths:
       - "litellm-rust/**"
+      - "litellm/rust_bridge/**"
+      - "tests/test_litellm_rust/**"
+      - "litellm/integrations/custom_logger.py"
+      - "litellm/litellm_core_utils/litellm_logging.py"
+      - "litellm/litellm_core_utils/logging_worker.py"
+      - "litellm/proxy/guardrails/**"
+      - "litellm/utils.py"
+      - "litellm/ocr/**"
+      - "litellm/llms/base_llm/ocr/**"
+      - "litellm/llms/custom_httpx/llm_http_handler.py"
+      - "tests/test_litellm/ocr/**"
+      - "tests/test_litellm/conftest.py"
+      - "Makefile"
       - ".cargo/**"
       - "pyproject.toml"
+      - "uv.lock"
       - "rust-toolchain.toml"
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/scripts/smoke_test_native_wheel.py"
@@ -20,8 +34,22 @@ on:
       - "litellm_**"
     paths:
       - "litellm-rust/**"
+      - "litellm/rust_bridge/**"
+      - "tests/test_litellm_rust/**"
+      - "litellm/integrations/custom_logger.py"
+      - "litellm/litellm_core_utils/litellm_logging.py"
+      - "litellm/litellm_core_utils/logging_worker.py"
+      - "litellm/proxy/guardrails/**"
+      - "litellm/utils.py"
+      - "litellm/ocr/**"
+      - "litellm/llms/base_llm/ocr/**"
+      - "litellm/llms/custom_httpx/llm_http_handler.py"
+      - "tests/test_litellm/ocr/**"
+      - "tests/test_litellm/conftest.py"
+      - "Makefile"
       - ".cargo/**"
       - "pyproject.toml"
+      - "uv.lock"
       - "rust-toolchain.toml"
       - ".github/actions/setup-uv-with-retries/**"
       - ".github/scripts/smoke_test_native_wheel.py"
@@ -42,7 +70,7 @@ env:
 jobs:
   rust-lint:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 15
     defaults:
       run:
         working-directory: litellm-rust
@@ -53,28 +81,48 @@ jobs:
 
       - run: rustup toolchain install --no-self-update
 
-      - run: cargo fmt --check
+      - run: cargo fmt --all --check
 
-      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            litellm-rust/target
-          key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-${{ github.job }}-
+          workspaces: litellm-rust
+          cache-on-failure: true
 
       - run: cargo clippy --workspace --all-targets --locked -- -D warnings
 
-      - run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
-
-      - run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings
-
   rust-test:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 20
+    defaults:
+      run:
+        working-directory: litellm-rust
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - run: rustup toolchain install --no-self-update
+
+      - uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8
+        with:
+          tool: cargo-nextest@0.9.143
+
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+        with:
+          workspaces: litellm-rust
+          cache-on-failure: true
+
+      - run: cargo nextest run --workspace --locked
+
+      - run: cargo test --workspace --doc --locked
+
+  rust-wheel:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
@@ -90,24 +138,10 @@ jobs:
 
       - run: rustup toolchain install --no-self-update
 
-      - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
         with:
-          path: |
-            ~/.cargo/registry
-            ~/.cargo/git
-            litellm-rust/target
-          key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-${{ github.job }}-
-
-      - run: cargo test --workspace --locked
-        working-directory: litellm-rust
-
-      - run: cargo test -p litellm-core --features bedrock-auth --locked
-        working-directory: litellm-rust
-
-      - run: cargo test -p litellm-ai-gateway --features server --locked
-        working-directory: litellm-rust
+          workspaces: litellm-rust
+          cache-on-failure: true
 
       - run: uv build --wheel --out-dir dist
 

.github/workflows/test-terraform-modules.yml

@@ -25,13 +25,17 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  aws-module:
-    name: fmt, validate, test (aws)
+  module:
+    name: fmt, validate, test (${{ matrix.module }})
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        module: [aws, gcp]
     defaults:
       run:
-        working-directory: terraform/litellm/aws
+        working-directory: terraform/litellm/${{ matrix.module }}
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
@@ -51,35 +55,7 @@ jobs:
       - name: validate
         run: terraform validate
 
-      # Plan-only, mock_provider-backed: no AWS credentials, no API calls.
+      # Plan-only, mock_provider-backed: no cloud credentials, no API calls.
       - name: test
         run: terraform test
 
-  gcp-module:
-    name: fmt, validate, test (gcp)
-    runs-on: ubuntu-latest
-    timeout-minutes: 15
-    defaults:
-      run:
-        working-directory: terraform/litellm/gcp
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
-        with:
-          terraform_version: 1.13.3
-          terraform_wrapper: false
-
-      - name: fmt
-        run: terraform fmt -recursive -check -diff
-
-      - name: init
-        run: terraform init -backend=false -input=false
-
-      - name: validate
-        run: terraform validate
-
-      - name: test
-        run: terraform test

.github/workflows/test-unit-proxy-db.yml

@@ -94,7 +94,6 @@ jobs:
               tests/proxy_unit_tests/test_jwt_key_mapping.py
               tests/proxy_unit_tests/test_proxy_custom_auth.py
               tests/proxy_unit_tests/test_key_generate_dynamodb.py
-              tests/proxy_unit_tests/test_deployed_proxy_keygen.py
             workers: 4
             dist: loadscope
             timeout: 15
@@ -110,8 +109,6 @@ jobs:
           - test-group: proxy-server-core
             test-path: >-
               tests/proxy_unit_tests/test_proxy_server.py
-              tests/proxy_unit_tests/test_proxy_server_keys.py
-              tests/proxy_unit_tests/test_proxy_server_spend.py
               tests/proxy_unit_tests/test_aproxy_startup.py
             workers: 4
             dist: loadscope
@@ -120,7 +117,6 @@ jobs:
             test-path: >-
               tests/proxy_unit_tests/test_proxy_config_unit_test.py
               tests/proxy_unit_tests/test_proxy_routes.py
-              tests/proxy_unit_tests/test_proxy_gunicorn.py
               tests/proxy_unit_tests/test_server_root_path.py
               tests/proxy_unit_tests/test_proxy_pass_user_config.py
               tests/proxy_unit_tests/test_proxy_token_counter.py
@@ -198,7 +194,6 @@ jobs:
               tests/proxy_unit_tests/test_realtime_cache.py
               tests/proxy_unit_tests/test_proxy_exception_mapping.py
               tests/proxy_unit_tests/test_custom_tokenizer_bug.py
-              tests/proxy_unit_tests/test_model_response_typing
             workers: 4
             dist: loadscope
             timeout: 15

.github/workflows/test-unit.yml

@@ -100,6 +100,7 @@ jobs:
               tests/test_litellm/secret_managers
               tests/test_litellm/a2a_protocol
               tests/test_litellm/anthropic_interface
+              tests/test_litellm/chat_completions
               tests/test_litellm/completion_extras
               tests/test_litellm/compression
               tests/test_litellm/containers
@@ -109,6 +110,7 @@ jobs:
               tests/test_litellm/repositories
               tests/test_litellm/images
               tests/test_litellm/interactions
+              tests/test_litellm/messages
               tests/test_litellm/ocr
               tests/test_litellm/passthrough
               tests/test_litellm/rag
@@ -200,6 +202,7 @@ jobs:
               tests/test_litellm/proxy/types_utils
               tests/test_litellm/proxy/logging_endpoints
               tests/test_litellm/proxy/test_*.py
+              tests/test_gateway
             workers: 4
             reruns: 2
             timeout-minutes: 20
@@ -210,7 +213,6 @@ jobs:
             test-path: >-
               tests/local_testing/test_cache_preset_key.py
               tests/local_testing/test_caching_handler.py
-              tests/local_testing/test_prompt_caching.py
               tests/local_testing/test_responses_stream_cache_keys.py
               tests/local_testing/test_unit_test_caching.py
             workers: 2

.gitignore

@@ -147,3 +147,6 @@ crash.*.log
 
 ui/litellm-dashboard/out/
 litellm.log
+
+.coverage-rust
+coverage-rust.xml

.grype.yaml

@@ -0,0 +1,13 @@
+# Wolfi's security database names zlib 1.3.3-r0 as the fix for CVE-2026-85091,
+# but the newest zlib published to the Wolfi apk repo is 1.3.2-r7, so every
+# wolfi-base digest reports it and no `apk upgrade` can clear it.
+# Drop this once Wolfi ships zlib >= 1.3.3-r0; expected by 2026-10-15.
+ignore:
+  - vulnerability: CVE-2026-85091
+    package:
+      name: zlib
+      type: apk
+  - vulnerability: GHSA-g5fp-32jq-cfw2
+    package:
+      name: zlib
+      type: apk

AGENTS.md

@@ -1 +1,3 @@
 Read @CLAUDE.md for coding guidelines
+
+Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or lint budgets to satisfy a check

ARCHITECTURE.md

@@ -45,7 +45,7 @@ sequenceDiagram
     ProxyServer->>Auth: user_api_key_auth()
     Auth->>Redis: Check API key cache
     Redis-->>Auth: Key info + spend limits
-    ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
+    ProxyServer->>Hooks: parallel_request_limiter, cache_control_check
     Hooks->>Redis: Check/increment rate limit counters
     ProxyServer->>Router: route_request()
     Router->>Main: litellm.acompletion()
@@ -145,7 +145,6 @@ graph TD
 
 | Hook | File | Purpose |
 |------|------|---------|
-| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits |
 | `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
 | `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
 | `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |

CLAUDE.md

@@ -25,6 +25,8 @@ Same thing for bug fixes. The tests should make it so that this specific bug can
 
 Never test structure of code only function of it
 
+A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
+
 `tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_<filename>.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_<filename>.py` if you're the first test there). One focused regression test beats many shallow ones
 
 End-to-end tests belong in `tests/e2e/` and must follow the harness conventions documented in that directory's `CLAUDE.md`

Dockerfile

@@ -8,9 +8,25 @@ ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
+# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
+ARG PGBOUNCER_VERSION=1.25.2
+ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
 
 FROM $UV_IMAGE AS uvbin
 
+FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder
+ARG PGBOUNCER_VERSION
+ARG PGBOUNCER_SHA256
+USER root
+RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl
+WORKDIR /build
+RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \
+    echo "${PGBOUNCER_SHA256}  pgbouncer.tar.gz" | sha256sum -c - && \
+    tar xzf pgbouncer.tar.gz --strip-components=1 && \
+    ./configure --prefix=/usr/local --with-openssl=/usr && \
+    make -j"$(nproc)" pgbouncer && \
+    install -m 0755 pgbouncer /usr/local/bin/pgbouncer
+
 # Admin UI builder. Pinned to the build platform so the architecture-independent
 # Next.js static export compiles once natively even in a multi-arch build,
 # instead of once per target arch under QEMU.
@@ -110,7 +126,8 @@ USER root
 RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
 
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
+RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
+COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 
 WORKDIR /app
 ENV PATH="/app/.venv/bin:${PATH}" \

Makefile

@@ -150,8 +150,8 @@ lint-install:
 # Diff-scoped format check, mirroring test-linting.yml's "Check ruff format" step:
 # only the litellm Python files changed vs the base are checked, so a pre-existing
 # format issue elsewhere doesn't block an unrelated commit. Git pathspecs match
-# recursively, so 'litellm/*.py' covers nested modules and the top-level files that
-# CI's 'litellm/**/*.py' skips, which makes this target a superset of the CI step.
+# recursively, so 'litellm/*.py' covers top-level files and nested modules alike,
+# the same set CI's ':(glob)litellm/**/*.py' selects.
 lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 	@base_ref=$$($(RESOLVE_BASE)) && \
 	changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
@@ -299,6 +299,9 @@ test-rust-extension:
 	[ "$$#" -eq 1 ] && \
 	UV_PROJECT_ENVIRONMENT="$$temporary/venv" $(UV) sync --python 3.12 --frozen --no-install-project --all-groups --all-extras && \
 	$(UV) pip install --python "$$temporary/venv/bin/python" --no-deps "$$1" && \
+	"$$temporary/venv/bin/python" -I -m mypy.stubtest \
+		--mypy-config-file tests/test_litellm/rust_bridge/stubtest.ini \
+		litellm.rust_bridge._native && \
 	LITELLM_RUST=1 LITELLM_LOCAL_MODEL_COST_MAP=True \
 	"$$temporary/venv/bin/python" -I -m pytest --import-mode=importlib -m requires_rust_extension tests/test_litellm_rust
 

README.md

@@ -262,6 +262,8 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 }
 ```
 
+For MCP OAuth, an upstream may advertise dynamic client registration but refuse requests with HTTP 401 or 403. If the provider requires a pre-registered OAuth app, configure its `credentials.client_id` and, when required, `credentials.client_secret` on the MCP server. This skips dynamic registration in the gateway sign-in flow. The provider must approve the app for MCP access; reaching its authorization page does not establish that login or tool calls will succeed
+
 [**Docs: MCP Gateway**](https://docs.litellm.ai/docs/mcp)
 
 </details>

cookbook/litellm_proxy_server/cli_token_usage.py

@@ -3,7 +3,7 @@
 Example: Using CLI token with LiteLLM SDK
 
 This example shows how to use the CLI authentication token
-in your Python scripts after running `litellm-proxy login`.
+in your Python scripts after running `lite login`.
 """
 
 from textwrap import indent
@@ -22,7 +22,7 @@ def main():
     api_key = litellm.get_litellm_gateway_api_key()
 
     if not api_key:
-        print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
+        print("❌ No CLI token found. Please run 'lite login' first.")
         return
 
     print("✅ Found CLI token.")
@@ -58,6 +58,6 @@ if __name__ == "__main__":
     main()
 
     print("\n💡 Tips:")
-    print("1. Run 'litellm-proxy login' to authenticate first")
+    print("1. Run 'lite login' to authenticate first")
     print("2. Replace 'https://your-proxy.com' with your actual proxy URL")
     print("3. The token is stored in your OS keychain, or in ~/.litellm/token.json when there is none")

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_1/grafana_dashboard.json

@@ -1,614 +0,0 @@
-{
-    "annotations": {
-      "list": [
-        {
-          "builtIn": 1,
-          "datasource": {
-            "type": "grafana",
-            "uid": "-- Grafana --"
-          },
-          "enable": true,
-          "hide": true,
-          "iconColor": "rgba(0, 211, 255, 1)",
-          "name": "Annotations & Alerts",
-          "target": {
-            "limit": 100,
-            "matchAny": false,
-            "tags": [],
-            "type": "dashboard"
-          },
-          "type": "dashboard"
-        }
-      ]
-    },
-    "description": "",
-    "editable": true,
-    "fiscalYearStartMonth": 0,
-    "graphTooltip": 0,
-    "id": 2039,
-    "links": [],
-    "liveNow": false,
-    "panels": [
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 0,
-              "gradientMode": "none",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "lineInterpolation": "linear",
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                },
-                {
-                  "color": "red",
-                  "value": 80
-                }
-              ]
-            },
-            "unit": "s"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 8,
-          "w": 12,
-          "x": 0,
-          "y": 0
-        },
-        "id": 10,
-        "options": {
-          "legend": {
-            "calcs": [],
-            "displayMode": "list",
-            "placement": "bottom",
-            "showLegend": true
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "${DS_PROMETHEUS}"
-            },
-            "editorMode": "code",
-            "expr": "histogram_quantile(0.99, sum(rate(litellm_self_latency_bucket{self=\"self\"}[1m])) by (le))",
-            "legendFormat": "Time to first token",
-            "range": true,
-            "refId": "A"
-          }
-        ],
-        "title": "Time to first token (latency)",
-        "type": "timeseries"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 0,
-              "gradientMode": "none",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "lineInterpolation": "linear",
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                },
-                {
-                  "color": "red",
-                  "value": 80
-                }
-              ]
-            },
-            "unit": "currencyUSD"
-          },
-          "overrides": [
-            {
-              "matcher": {
-                "id": "byName",
-                "options": "7e4b0627fd32efdd2313c846325575808aadcf2839f0fde90723aab9ab73c78f"
-              },
-              "properties": [
-                {
-                  "id": "displayName",
-                  "value": "Translata"
-                }
-              ]
-            }
-          ]
-        },
-        "gridPos": {
-          "h": 8,
-          "w": 12,
-          "x": 0,
-          "y": 8
-        },
-        "id": 11,
-        "options": {
-          "legend": {
-            "calcs": [],
-            "displayMode": "list",
-            "placement": "bottom",
-            "showLegend": true
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "${DS_PROMETHEUS}"
-            },
-            "editorMode": "code",
-            "expr": "sum(increase(litellm_spend_metric_total[30d])) by (hashed_api_key)",
-            "legendFormat": "{{team}}",
-            "range": true,
-            "refId": "A"
-          }
-        ],
-        "title": "Spend by team",
-        "transformations": [],
-        "type": "timeseries"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 0,
-              "gradientMode": "none",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "lineInterpolation": "linear",
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                },
-                {
-                  "color": "red",
-                  "value": 80
-                }
-              ]
-            }
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 9,
-          "w": 12,
-          "x": 0,
-          "y": 16
-        },
-        "id": 2,
-        "options": {
-          "legend": {
-            "calcs": [],
-            "displayMode": "list",
-            "placement": "bottom",
-            "showLegend": true
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "${DS_PROMETHEUS}"
-            },
-            "editorMode": "code",
-            "expr": "sum by (model) (increase(litellm_requests_metric_total[5m]))",
-            "legendFormat": "{{model}}",
-            "range": true,
-            "refId": "A"
-          }
-        ],
-        "title": "Requests by model",
-        "type": "timeseries"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "thresholds"
-            },
-            "mappings": [],
-            "noValue": "0",
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                },
-                {
-                  "color": "red",
-                  "value": 80
-                }
-              ]
-            }
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 7,
-          "w": 3,
-          "x": 0,
-          "y": 25
-        },
-        "id": 8,
-        "options": {
-          "colorMode": "value",
-          "graphMode": "area",
-          "justifyMode": "auto",
-          "orientation": "auto",
-          "reduceOptions": {
-            "calcs": [
-              "lastNotNull"
-            ],
-            "fields": "",
-            "values": false
-          },
-          "textMode": "auto"
-        },
-        "pluginVersion": "9.4.17",
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "${DS_PROMETHEUS}"
-            },
-            "editorMode": "code",
-            "expr": "sum(increase(litellm_llm_api_failed_requests_metric_total[1h]))",
-            "legendFormat": "__auto",
-            "range": true,
-            "refId": "A"
-          }
-        ],
-        "title": "Faild Requests",
-        "type": "stat"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 0,
-              "gradientMode": "none",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "lineInterpolation": "linear",
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                },
-                {
-                  "color": "red",
-                  "value": 80
-                }
-              ]
-            },
-            "unit": "currencyUSD"
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 7,
-          "w": 3,
-          "x": 3,
-          "y": 25
-        },
-        "id": 6,
-        "options": {
-          "legend": {
-            "calcs": [],
-            "displayMode": "list",
-            "placement": "bottom",
-            "showLegend": true
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "${DS_PROMETHEUS}"
-            },
-            "editorMode": "code",
-            "expr": "sum(increase(litellm_spend_metric_total[30d])) by (model)",
-            "legendFormat": "{{model}}",
-            "range": true,
-            "refId": "A"
-          }
-        ],
-        "title": "Spend",
-        "type": "timeseries"
-      },
-      {
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${DS_PROMETHEUS}"
-        },
-        "fieldConfig": {
-          "defaults": {
-            "color": {
-              "mode": "palette-classic"
-            },
-            "custom": {
-              "axisCenteredZero": false,
-              "axisColorMode": "text",
-              "axisLabel": "",
-              "axisPlacement": "auto",
-              "barAlignment": 0,
-              "drawStyle": "line",
-              "fillOpacity": 0,
-              "gradientMode": "none",
-              "hideFrom": {
-                "legend": false,
-                "tooltip": false,
-                "viz": false
-              },
-              "lineInterpolation": "linear",
-              "lineWidth": 1,
-              "pointSize": 5,
-              "scaleDistribution": {
-                "type": "linear"
-              },
-              "showPoints": "auto",
-              "spanNulls": false,
-              "stacking": {
-                "group": "A",
-                "mode": "none"
-              },
-              "thresholdsStyle": {
-                "mode": "off"
-              }
-            },
-            "mappings": [],
-            "thresholds": {
-              "mode": "absolute",
-              "steps": [
-                {
-                  "color": "green",
-                  "value": null
-                },
-                {
-                  "color": "red",
-                  "value": 80
-                }
-              ]
-            }
-          },
-          "overrides": []
-        },
-        "gridPos": {
-          "h": 7,
-          "w": 6,
-          "x": 6,
-          "y": 25
-        },
-        "id": 4,
-        "options": {
-          "legend": {
-            "calcs": [],
-            "displayMode": "list",
-            "placement": "bottom",
-            "showLegend": true
-          },
-          "tooltip": {
-            "mode": "single",
-            "sort": "none"
-          }
-        },
-        "targets": [
-          {
-            "datasource": {
-              "type": "prometheus",
-              "uid": "${DS_PROMETHEUS}"
-            },
-            "editorMode": "code",
-            "expr": "sum(increase(litellm_total_tokens_total[5m])) by (model)",
-            "legendFormat": "__auto",
-            "range": true,
-            "refId": "A"
-          }
-        ],
-        "title": "Tokens",
-        "type": "timeseries"
-      }
-    ],
-    "refresh": "1m",
-    "revision": 1,
-    "schemaVersion": 38,
-    "style": "dark",
-    "tags": [],
-    "templating": {
-      "list": [
-        {
-          "current": {
-            "selected": false,
-            "text": "prometheus",
-            "value": "edx8memhpd9tsa"
-          },
-          "hide": 0,
-          "includeAll": false,
-          "label": "datasource",
-          "multi": false,
-          "name": "DS_PROMETHEUS",
-          "options": [],
-          "query": "prometheus",
-          "queryValue": "",
-          "refresh": 1,
-          "regex": "",
-          "skipUrlSync": false,
-          "type": "datasource"
-        }
-      ]
-    },
-    "time": {
-      "from": "now-1h",
-      "to": "now"
-    },
-    "timepicker": {},
-    "timezone": "",
-    "title": "LLM Proxy",
-    "uid": "rgRrHxESz",
-    "version": 15,
-    "weekStart": ""
-  }

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_1/readme.md

@@ -1,6 +0,0 @@
-## This folder contains the `json` for creating the following Grafana Dashboard
-
-### Pre-Requisites
-- Setup LiteLLM Proxy Prometheus Metrics https://docs.litellm.ai/docs/proxy/prometheus 
-
-![1716623265684](https://github.com/BerriAI/litellm/assets/29436595/0e12c57e-4a2d-4850-bd4f-e4294f87a814)

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/readme.md

@@ -0,0 +1,11 @@
+# LiteLLM All Prometheus Metrics dashboard
+
+Every `litellm_*` metric family the proxy can expose on `/metrics` (134 families across 95 panels), grouped into rows: proxy traffic, latency, spend and tokens, cache, LLM API deployments, key and team rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, the Redis circuit breaker, the spend log cleanup job, and the `prometheus_system` service callback metrics (per-service latency, request and failure rates, spend update queue sizes). Panel titles are the metric names so you can grep the JSON for the metric you care about
+
+Import `grafana_dashboard.json` from **Dashboards > New > Import** and pick your Prometheus data source when prompted (the `DS_PROMETHEUS` variable). Counters are plotted as `rate()` over `$__rate_interval`, histograms as p50 / p95 / p99, gauges as the raw value grouped by the most useful label. Every query names the metric exactly as the proxy emits it (counters carry the `_total` suffix the Prometheus client adds), and `tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py` fails if a metric is renamed without updating this dashboard
+
+The first eleven rows need only `callbacks: ["prometheus"]`. The last three rows and the `litellm_admission_*` panels are emitted by other subsystems and stay empty until those are on: the service callback row needs `service_callback: ["prometheus_system"]` in `litellm_settings`, the circuit breaker row needs a Redis cache, the cleanup row needs spend log retention, and admission control needs its middleware enabled. Within the base rows, many panels only fill in once the matching feature is in use: budgets need keys, teams, users or orgs with `max_budget` set, cache panels need caching on, guardrail and MCP panels need those features configured, deployment health needs the router with more than one deployment or a failure to record, and `litellm_in_flight_requests` needs traffic at scrape time. An empty panel for a feature you do not use is expected
+
+## Pre-requisites
+
+Prometheus metrics on the proxy: https://docs.litellm.ai/docs/proxy/prometheus

cookbook/litellm_proxy_server/grafana_dashboard/dashboard_v2/grafana_dashboard.json

@@ -476,7 +476,7 @@
               "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
-            "expr": "topk(5, sort(litellm_remaining_requests))",
+            "expr": "topk(5, sort(litellm_remaining_requests_metric))",
             "legendFormat": "__auto",
             "range": true,
             "refId": "A"
@@ -573,7 +573,7 @@
               "uid": "${DS_PROMETHEUS}"
             },
             "editorMode": "code",
-            "expr": "topk(5, sort(litellm_remaining_tokens))",
+            "expr": "topk(5, sort(litellm_remaining_tokens_metric))",
             "legendFormat": "__auto",
             "range": true,
             "refId": "A"

cookbook/litellm_proxy_server/grafana_dashboard/readme.md

@@ -6,8 +6,14 @@ This folder contains the `json` for creating Grafana Dashboards
 
 Charts the `gen_ai.*` metrics from the OpenTelemetry v2 integration: spend, tokens, request rate, and latency percentiles by model. Separate from the dashboards below, which chart the `litellm_*` Prometheus metrics.
 
+## [LiteLLM All Prometheus Metrics dashboard](./dashboard_all_metrics)
+
+Every `litellm_*` Prometheus metric family the proxy can emit (134 families, 95 panels) grouped by theme: traffic, latency, spend and tokens, cache, deployments, rate limits, budgets, guardrails, MCP, managed files and batches, users and teams, plus the Redis circuit breaker, spend log cleanup and `prometheus_system` service metrics. Start here if you want everything on one screen; see its [readme](./dashboard_all_metrics/readme.md) for import steps and which panels need a feature enabled before they show data
+
 ## [LiteLLM v2 Dashboard](./dashboard_v2)
 
+A compact view of proxy request rate, failures, latency and the top remaining-request / remaining-token gauges per model group
+
 <img width="1316" alt="grafana_1" src="https://github.com/user-attachments/assets/d0df802d-0cb9-4906-a679-941c547789ab">
 <img width="1289" alt="grafana_2" src="https://github.com/user-attachments/assets/b11f755f-e113-42ab-b21d-83f91f451a28">
 <img width="1323" alt="grafana_3" src="https://github.com/user-attachments/assets/cb29ffdb-477d-4be1-a5cd-c3f7f2cb21c5">

db_scripts/partition_spend_logs.sql

@@ -53,6 +53,8 @@ ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_end_user_idx"
     RENAME TO "LiteLLM_SpendLogs_legacy_end_user_idx";
 ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_session_id_idx"
     RENAME TO "LiteLLM_SpendLogs_legacy_session_id_idx";
+ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx"
+    RENAME TO "LiteLLM_SpendLogs_legacy_api_key_startTime_idx";
 
 CREATE TABLE "LiteLLM_SpendLogs" (
     LIKE "LiteLLM_SpendLogs_legacy" INCLUDING DEFAULTS INCLUDING GENERATED
@@ -78,6 +80,9 @@ CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_end_user_idx"
 CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_session_id_idx"
     ON "LiteLLM_SpendLogs" ("session_id");
 
+CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx"
+    ON "LiteLLM_SpendLogs" ("api_key", "startTime");
+
 -- Safety net: any row whose startTime has no explicit partition lands here so
 -- writes never fail. The cleanup job never drops the DEFAULT partition.
 CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogs_pdefault"

db_scripts/unpartition_spend_logs.sql

@@ -40,6 +40,8 @@ ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_end_user_idx"
     RENAME TO "LiteLLM_SpendLogs_partitioned_end_user_idx";
 ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_session_id_idx"
     RENAME TO "LiteLLM_SpendLogs_partitioned_session_id_idx";
+ALTER INDEX IF EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx"
+    RENAME TO "LiteLLM_SpendLogs_partitioned_api_key_startTime_idx";
 
 CREATE TABLE "LiteLLM_SpendLogs" (
     LIKE "LiteLLM_SpendLogs_partitioned" INCLUDING DEFAULTS INCLUDING GENERATED
@@ -60,6 +62,9 @@ CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_end_user_idx"
 CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_session_id_idx"
     ON "LiteLLM_SpendLogs" ("session_id");
 
+CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx"
+    ON "LiteLLM_SpendLogs" ("api_key", "startTime");
+
 INSERT INTO "LiteLLM_SpendLogs"
 SELECT * FROM "LiteLLM_SpendLogs_partitioned"
 ON CONFLICT ("request_id") DO NOTHING;

docker/Dockerfile.database

@@ -8,9 +8,25 @@ ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
+# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
+ARG PGBOUNCER_VERSION=1.25.2
+ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
 
 FROM $UV_IMAGE AS uvbin
 
+FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder
+ARG PGBOUNCER_VERSION
+ARG PGBOUNCER_SHA256
+USER root
+RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl
+WORKDIR /build
+RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \
+    echo "${PGBOUNCER_SHA256}  pgbouncer.tar.gz" | sha256sum -c - && \
+    tar xzf pgbouncer.tar.gz --strip-components=1 && \
+    ./configure --prefix=/usr/local --with-openssl=/usr && \
+    make -j"$(nproc)" pgbouncer && \
+    install -m 0755 pgbouncer /usr/local/bin/pgbouncer
+
 # Admin UI builder. Pinned to the build platform so the architecture-independent
 # Next.js static export compiles once natively even in a multi-arch build,
 # instead of once per target arch under QEMU.
@@ -101,7 +117,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # node (without npm) is required by the prisma CLI at runtime
-RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile
+RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
+COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 
 WORKDIR /app
 ENV PATH="/app/.venv/bin:${PATH}" \

docker/Dockerfile.non_root

@@ -7,9 +7,25 @@ ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
+# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
+ARG PGBOUNCER_VERSION=1.25.2
+ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
 
 FROM $UV_IMAGE AS uvbin
 
+FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder
+ARG PGBOUNCER_VERSION
+ARG PGBOUNCER_SHA256
+USER root
+RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl
+WORKDIR /build
+RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \
+    echo "${PGBOUNCER_SHA256}  pgbouncer.tar.gz" | sha256sum -c - && \
+    tar xzf pgbouncer.tar.gz --strip-components=1 && \
+    ./configure --prefix=/usr/local --with-openssl=/usr && \
+    make -j"$(nproc)" pgbouncer && \
+    install -m 0755 pgbouncer /usr/local/bin/pgbouncer
+
 # Admin UI builder. Pinned to the build platform so the architecture-independent
 # Next.js static export compiles once natively even in a multi-arch build,
 # instead of once per target arch under QEMU.
@@ -128,8 +144,9 @@ RUN for i in 1 2 3; do \
       apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
+      apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \
     done
+COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 
 # Copy only what runtime needs. The application is installed inside the venv;
 # the rest of the builder's /app is source and build metadata that must not

enterprise/litellm_enterprise/enterprise_callbacks/secret_detection.py

@@ -11,10 +11,14 @@ import sys
 sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
-import functools
+import configparser
+import contextlib
+import itertools
+import re
 import tempfile
+from collections.abc import Generator, Iterator, Sequence
 from contextvars import ContextVar
-from typing import TYPE_CHECKING, ClassVar, Literal, Optional
+from typing import TYPE_CHECKING, ClassVar, Final, Literal, Optional
 
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
@@ -433,12 +437,101 @@ _default_detect_secrets_config = {
             "name": "ZendeskSecretKeyDetector",
             "path": _custom_plugins_path + "/zendesk_secret_key.py",
         },
+        {
+            "name": "CredentialKeywordDetector",
+            "path": _custom_plugins_path + "/credential_keyword.py",
+        },
         {"name": "Base64HighEntropyString", "limit": 4.5},
         {"name": "HexHighEntropyString", "limit": 3.0},
     ],
 }
 
 
+_CONFIG_SECTION: Final = "litellm-prompt"
+
+_ASSIGNMENT_LINE: Final = re.compile(r"[^\s\[#;:=][^:=]*[:=]")
+
+_SHELL_ASSIGNMENT: Final = re.compile(r"(?P<key>[^\s\[#;:=](?:[^:=]*[^\s:=])?)=(?P<value>\S+)")
+
+_SHELL_OPERATORS: Final = ";&|"
+
+_SHELL_TRAILER: Final = re.compile(r"\\|#.*|-*\w[\w.-]*=\S*")
+
+_SCAN_SUFFIX: Final = ".py"
+
+
+@contextlib.contextmanager
+def _temp_file(text: str) -> Generator[str, None, None]:
+    temp_file: Final = tempfile.NamedTemporaryFile(suffix=_SCAN_SUFFIX, delete=False)
+    try:
+        temp_file.write(text.encode("utf-8"))
+        temp_file.close()
+        yield temp_file.name
+    finally:
+        temp_file.close()
+        os.remove(temp_file.name)
+
+
+def _scan_lines(lines: Sequence[str]) -> frozenset[tuple[str, str]]:
+    from detect_secrets import SecretsCollection
+
+    secrets: Final = SecretsCollection()
+    with _temp_file("\n".join(lines)) as path:
+        secrets.scan_file(path)
+
+    return frozenset(
+        (found_secret.secret_value, found_secret.type)
+        for file in secrets.files
+        for found_secret in secrets[file]
+        if found_secret.secret_value is not None
+    )
+
+
+def _classify_line(state: tuple[bool, str | None], numbered: tuple[int, str]) -> tuple[bool, str | None]:
+    open_option: Final = state[0]
+    number, line = numbered
+    stripped: Final = line.strip()
+    if not stripped or stripped[0] in "#;":
+        return open_option, None
+    shell_assignment: Final = _SHELL_ASSIGNMENT.match(stripped)
+    if shell_assignment is not None:
+        return True, f"{shell_assignment['key']}_{number}={shell_assignment['value']}"
+    assignment: Final = _ASSIGNMENT_LINE.match(stripped)
+    if assignment is not None:
+        return True, f"{assignment.group()[:-1].strip()}_{number}{stripped[assignment.end() - 1 :]}"
+    if line[0].isspace() and open_option:
+        return True, line
+    return False, None
+
+
+def _parseable_lines(text: str) -> Iterator[str]:
+    states: Final = itertools.accumulate(enumerate(text.splitlines()), _classify_line, initial=(False, None))
+    return (line for _, line in states if line is not None)
+
+
+def _lone_value(line: str) -> str | None:
+    tokens: Final = line.split()
+    if not tokens or '"' in tokens[0]:
+        return None
+    value: Final = tokens[0].rstrip(_SHELL_OPERATORS)
+    if len(tokens) == 1 or value != tokens[0] or _SHELL_TRAILER.fullmatch(tokens[1]) is not None:
+        return value
+    return None
+
+
+def _quoted_assignments(text: str) -> tuple[str, ...]:
+    parser: Final = configparser.ConfigParser(interpolation=None)
+    parser.optionxform = str  # pyright: ignore[reportAttributeAccessIssue]  # configparser types optionxform as a method
+    parser.read_string(f"[{_CONFIG_SECTION}]\n" + "\n".join(_parseable_lines(text)))
+    return tuple(
+        f'{key} = "{value}"'
+        for section in parser
+        for key, values in parser.items(section)
+        for line in values.splitlines()
+        if (value := _lone_value(line)) is not None
+    )
+
+
 class _ENTERPRISE_SecretDetection(CustomGuardrail):
     # Keeps proxied traffic on async_pre_call_hook (the unified apply_guardrail
     # path skips should_run_check and never sees data["prompt"]).
@@ -449,35 +542,21 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
         super().__init__(**kwargs)
 
     def scan_message_for_secrets(self, message_content: str):
-        from detect_secrets import SecretsCollection
         from detect_secrets.settings import transient_settings
 
-        temp_file = tempfile.NamedTemporaryFile(delete=False)
-        temp_file.write(message_content.encode("utf-8"))
-        temp_file.close()
-
-        secrets = SecretsCollection()
-
         detect_secrets_config = (
             self.user_defined_detect_secrets_config or _default_detect_secrets_config
         )
         with transient_settings(detect_secrets_config):
-            secrets.scan_file(temp_file.name)
-
-        os.remove(temp_file.name)
+            found: Final = _scan_lines(
+                (*message_content.splitlines(), *_quoted_assignments(message_content))
+            )
 
         return [
-            {"type": found_secret.type, "value": found_secret.secret_value}
-            for file in sorted(secrets.files)
-            for found_secret in sorted(
-                secrets[file],
-                key=lambda secret: (
-                    -len(secret.secret_value or ""),
-                    secret.type,
-                    secret.secret_value or "",
-                ),
+            {"type": secret_type, "value": value}
+            for value, secret_type in sorted(
+                found, key=lambda pair: (-len(pair[0]), pair[1], pair[0])
             )
-            if found_secret.secret_value is not None
         ]
 
     def redact_text(self, text: str, source: str = "message") -> str:
@@ -490,15 +569,16 @@ class _ENTERPRISE_SecretDetection(CustomGuardrail):
         if counts is not None:
             for secret in detected_secrets:
                 counts[secret["type"]] = counts.get(secret["type"], 0) + 1
-        secret_types = [secret["type"] for secret in detected_secrets]
+        secret_types: Final = sorted(
+            dict.fromkeys(secret["type"] for secret in detected_secrets)
+        )
         verbose_proxy_logger.warning(
-            f"Detected and redacted secrets in {source}: {secret_types}"
+            "Detected and redacted secrets in %s: %s", source, secret_types
         )
-        return functools.reduce(
-            lambda redacted, secret: redacted.replace(secret["value"], "[REDACTED]"),
-            detected_secrets,
-            text,
+        pattern: Final = re.compile(
+            "|".join(re.escape(secret["value"]) for secret in detected_secrets)
         )
+        return pattern.sub("[REDACTED]", text)
 
     async def should_run_check(self, user_api_key_dict: UserAPIKeyAuth) -> bool:
         if user_api_key_dict.permissions is not None:

enterprise/litellm_enterprise/enterprise_callbacks/secrets_plugins/credential_keyword.py

@@ -0,0 +1,63 @@
+import re
+from collections.abc import Generator, Mapping
+from string import punctuation
+from typing import Final
+
+from detect_secrets.plugins.keyword import (
+    QUOTES_REQUIRED_DENYLIST_REGEX_TO_GROUP,
+    KeywordDetector,
+)
+
+_CREDENTIAL_VALUE: Final = re.compile(r"[^\s()\[\]]+")
+_ENVIRONMENT_REFERENCE: Final = re.compile(r"os\.environ/\w+", re.IGNORECASE)
+_ENVIRONMENT_VARIABLE_NAME: Final = re.compile(r"[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+")
+_LOWERCASE_WORD_SEQUENCE: Final = re.compile(r"[a-z]+(?:[-._/][a-z]+)+")
+_ISO_8601_TIMESTAMP: Final = re.compile(
+    r"\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?"
+)
+_URL_WITHOUT_USERINFO_OR_QUERY: Final = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*://[^\s@?]*")
+_BENIGN_VALUES: Final = (
+    _ENVIRONMENT_REFERENCE,
+    _ENVIRONMENT_VARIABLE_NAME,
+    _LOWERCASE_WORD_SEQUENCE,
+    _ISO_8601_TIMESTAMP,
+    _URL_WITHOUT_USERINFO_OR_QUERY,
+)
+
+
+class CredentialKeywordDetector(KeywordDetector):  # pyright: ignore[reportUntypedBaseClass]  # detect_secrets ships no type information
+    secret_type = "Credential Keyword"
+
+    def __init__(self, minimum_length: int = 12, keyword_exclude: str | None = None) -> None:
+        if (
+            not isinstance(minimum_length, int)  # pyright: ignore[reportUnnecessaryIsInstance]  # the value comes from an operator's YAML
+            or minimum_length < 1
+        ):
+            raise ValueError(f"minimum_length must be a positive integer, got {minimum_length!r}")
+        super().__init__(keyword_exclude=keyword_exclude)
+        self.minimum_length = minimum_length
+
+    def _is_credential(self, value: str) -> bool:
+        core: Final = value.strip(punctuation)
+        return (
+            len(value) >= self.minimum_length
+            and _CREDENTIAL_VALUE.fullmatch(value) is not None
+            and all(benign.fullmatch(core) is None for benign in _BENIGN_VALUES)
+        )
+
+    def analyze_string(
+        self,
+        string: str,
+        denylist_regex_to_group: Mapping[re.Pattern[str], int] | None = None,
+    ) -> Generator[str, None, None]:
+        if self.keyword_exclude is not None and self.keyword_exclude.search(string):
+            return
+        regex_to_group: Final = (
+            QUOTES_REQUIRED_DENYLIST_REGEX_TO_GROUP if denylist_regex_to_group is None else denylist_regex_to_group
+        )
+        yield from (
+            match.group(group)
+            for regex, group in regex_to_group.items()
+            for match in regex.finditer(string)
+            if self._is_credential(match.group(group))
+        )

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -27,10 +27,13 @@ import litellm
 from litellm import Router, verbose_logger
 from litellm._uuid import uuid
 from litellm.caching.caching import DualCache
+from litellm.constants import MAX_FILE_LIST_LIMIT
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_metadata,
 )
+from openai.types.file_deleted import FileDeleted
+
 from litellm.llms.base_llm.files.transformation import BaseFileEndpoints
 from litellm.llms.base_llm.managed_resources.isolation import (
     build_list_page,
@@ -48,7 +51,6 @@ from litellm.proxy._types import (
 from litellm.proxy.openai_files_endpoints.common_utils import (
     BATCH_CREATE_HIDDEN_PARAM,
     FILE_LIST_CONTINUATION_CHUNK_SIZE,
-    MAX_FILE_LIST_LIMIT,
     _is_base64_encoded_unified_file_id,
     apply_unified_file_ids,
     decode_model_from_file_id,
@@ -1787,7 +1789,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         litellm_parent_otel_span: Optional[Span],
         llm_router: Router,
         **data: Dict,
-    ) -> OpenAIFileObject:
+    ) -> FileDeleted:
 
         # Check if file deletion should be blocked due to batch references
         await self._check_file_deletion_allowed(file_id)
@@ -1795,7 +1797,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         # file_id = convert_b64_uid_to_unified_uid(file_id)
         model_file_id_mapping = await self.get_model_file_id_mapping([file_id], litellm_parent_otel_span)
 
-        delete_response = None
         specific_model_file_id_mapping = model_file_id_mapping.get(file_id)
         if specific_model_file_id_mapping:
             # Remove conflicting keys from data to avoid duplicate keyword arguments
@@ -1810,23 +1811,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                         else {}
                     ),
                 }
-                delete_response = await llm_router.afile_delete(model=model_id, file_id=model_file_id, **delete_data)
+                await llm_router.afile_delete(model=model_id, file_id=model_file_id, **delete_data)
 
-        stored_file_object = await self.delete_unified_file_id(file_id, litellm_parent_otel_span)
+        await self.delete_unified_file_id(file_id, litellm_parent_otel_span)
 
-        # Record successful deletion metric only on actual success
-        if stored_file_object or delete_response:
-            prom_logger = self._get_prometheus_logger()
-            if prom_logger:
-                prom_logger.record_managed_file_deleted(result="success")
-
-        if stored_file_object:
-            return OpenAIFileObject.model_validate(stored_file_object).model_copy(update={"id": file_id})
-        elif delete_response:
-            delete_response.id = file_id
-            return delete_response
-        else:
-            raise Exception(f"LiteLLM Managed File object with id={file_id} not found")
+        prom_logger = self._get_prometheus_logger()
+        if prom_logger:
+            prom_logger.record_managed_file_deleted(result="success")
+        return FileDeleted(id=file_id, object="file", deleted=True)
 
     async def afile_content(
         self,

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -780,7 +780,10 @@ async def update_project(
 
         # Handle budget updates
         budget_fields = LiteLLM_BudgetTable.model_fields.keys()
-        budget_updates = {k: v for k, v in update_data.items() if k in budget_fields}
+        budget_updates = {
+            **{k: v for k, v in update_data.items() if k in budget_fields},
+            **({"max_budget": None} if "max_budget" in data.model_fields_set and data.max_budget is None else {}),
+        }
 
         if budget_updates and existing_project.budget_id:
             # Update existing budget

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.66"
+version = "0.1.68"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.66"
+version = "0.1.68"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -1,9 +1,25 @@
 ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
+# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
+ARG PGBOUNCER_VERSION=1.25.2
+ARG PGBOUNCER_SHA256=924ad35113fd0a71c8e2dbe85b5d03445532e2b7b37a9f8a48983beea238b332
 
 FROM $UV_IMAGE AS uvbin
 
+FROM $LITELLM_BUILD_IMAGE AS pgbouncer-builder
+ARG PGBOUNCER_VERSION
+ARG PGBOUNCER_SHA256
+USER root
+RUN apk add --no-cache build-base pkgconf libevent-dev openssl-dev curl
+WORKDIR /build
+RUN curl -fsSL -o pgbouncer.tar.gz "https://www.pgbouncer.org/downloads/files/${PGBOUNCER_VERSION}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" && \
+    echo "${PGBOUNCER_SHA256}  pgbouncer.tar.gz" | sha256sum -c - && \
+    tar xzf pgbouncer.tar.gz --strip-components=1 && \
+    ./configure --prefix=/usr/local --with-openssl=/usr && \
+    make -j"$(nproc)" pgbouncer && \
+    install -m 0755 pgbouncer /usr/local/bin/pgbouncer
+
 # ---------- Builder ----------
 FROM $LITELLM_BUILD_IMAGE AS builder
 
@@ -61,6 +77,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra bedrock-realtime \
         --python python3.13
 
+# PYTHONPATH=/app makes the source tree shadow the installed package, so the
+# compiled Rust extension must live next to the source or it is never imported.
+RUN cp "$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"/litellm/rust_bridge/_native*.so litellm/rust_bridge/
+
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
     npm_config_cache=/root/.npm \
     prisma generate --schema=./schema.prisma
@@ -73,7 +93,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 RUN for i in 1 2 3; do \
-      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
+      apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \
       [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
       sleep 5; \
     done
@@ -90,15 +110,17 @@ ENV HOME=/home/nonroot \
 
 COPY --from=builder --chown=nonroot:nonroot /app /app
 COPY --from=builder /opt/prisma /opt/prisma
+COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete && \
     chmod -R a+rX /opt/prisma && \
-    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths"
+    python -c "from prisma.client import BINARY_PATHS; paths = list(BINARY_PATHS.query_engine.values()); assert paths and all(p.startswith('/opt/prisma/') for p in paths), paths" && \
+    python -c "import litellm; from litellm.rust_bridge.loader import native_bridge_available; assert litellm.__file__ == '/app/litellm/__init__.py', litellm.__file__; assert native_bridge_available()"
 
 USER nonroot
 
 EXPOSE 4000/tcp
 
-ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh uvicorn gateway.main:app --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
+ENTRYPOINT ["sh", "-c", "exec /app/docker/component_entrypoint.sh python -m gateway.launch --workers \"${NUM_WORKERS:-1}\" \"$@\"", "--"]
 CMD ["--host", "0.0.0.0", "--port", "4000"]

gateway/launch.py

@@ -0,0 +1,77 @@
+"""Gateway supervisor: assemble DATABASE_URL, start the in-container PgBouncer, then run uvicorn.
+
+``gateway/main.py`` assembles ``DATABASE_URL`` inside every uvicorn worker, which
+is fine for a plain Postgres URL but not for the pooler: PgBouncer must be
+started exactly once per pod, before the workers fork, and the workers must be
+handed the loopback URL it listens on. A pre-existing ``DATABASE_URL`` wins in
+``DatabaseURLSettings.apply_to_env`` under password auth, and one marked pooled
+wins under token auth too, so exporting it here is enough for every worker to
+pick the pooled URL up unchanged.
+
+Run with:
+    python -m gateway.launch --workers 4 --host 0.0.0.0 --port 4000
+"""
+
+import os
+import sys
+from collections.abc import Callable, Mapping, Sequence
+from typing import Final
+
+from uvicorn.main import main as uvicorn_main
+
+from litellm.proxy.db.db_url_settings import DatabaseURLSettings
+from litellm.proxy.db.pgbouncer import (
+    PgBouncerError,
+    PgBouncerSettings,
+    export_pooled_database_url,
+    start_in_container_pgbouncer,
+)
+
+GATEWAY_APP: Final = "gateway.main:app"
+KEEPALIVE_FLAG: Final = "--timeout-keep-alive"
+
+
+def uvicorn_argv(argv: Sequence[str], environ: Mapping[str, str]) -> tuple[str, ...]:
+    """Honor ``KEEPALIVE_TIMEOUT`` like ``proxy_cli.py`` does, unless the flag was passed explicitly."""
+    keepalive: Final = environ.get("KEEPALIVE_TIMEOUT")
+    if keepalive is None or any(arg == KEEPALIVE_FLAG or arg.startswith(f"{KEEPALIVE_FLAG}=") for arg in argv):
+        return (GATEWAY_APP, *argv)
+    return (GATEWAY_APP, *argv, KEEPALIVE_FLAG, keepalive)
+
+
+def pool_database_url(
+    settings: DatabaseURLSettings,
+    pgbouncer: PgBouncerSettings,
+    environ: Mapping[str, str],
+) -> str | PgBouncerError | None:
+    """Start the in-container PgBouncer and return its loopback URL, or None when ``pgbouncer.enabled`` is off.
+
+    The upstream URL is whatever ``apply_to_env`` assembled from the discrete
+    ``DATABASE_*`` vars (or an operator-pinned ``DATABASE_URL``). Under token
+    auth the pooler mints and renews the upstream token itself.
+    """
+    if not pgbouncer.enabled:
+        return None
+    upstream_url: Final = environ.get("DATABASE_URL")
+    if upstream_url is None:
+        return PgBouncerError("LITELLM_PGBOUNCER_ENABLED is set but no DATABASE_URL could be assembled")
+    return start_in_container_pgbouncer(pgbouncer, upstream_url, token_auth=settings.token_auth())
+
+
+def _serve(argv: Sequence[str]) -> None:
+    uvicorn_main(tuple(argv), prog_name="uvicorn")
+
+
+def main(argv: Sequence[str], serve: Callable[[Sequence[str]], None] = _serve) -> None:
+    settings: Final = DatabaseURLSettings.from_env()
+    settings.apply_to_env()
+    pooled_url: Final = pool_database_url(settings, PgBouncerSettings(), os.environ)
+    if isinstance(pooled_url, PgBouncerError):
+        sys.exit(f"LiteLLM gateway: in-container pgbouncer could not start: {pooled_url.reason}")
+    if pooled_url is not None:
+        export_pooled_database_url(pooled_url)
+    serve(uvicorn_argv(argv, os.environ))
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

gateway/routes/allowlist.py

@@ -3,7 +3,8 @@
 The gateway exposes the LLM data-plane surface: chat/completions, embeddings,
 audio, batches, files, fine-tuning, rerank, ocr, rag, video, search, image,
 responses, vector stores, passthrough providers, realtime websockets, MCP
-tool-call endpoints, and operational endpoints (/health, /metrics).
+tool-call endpoints, and operational endpoints (/health, /metrics, and the
+/debug/memory/summary read of the serving worker's RSS).
 
 Any path not listed here is dropped from the gateway process so management/UI
 endpoints don't ride on the same pods.
@@ -95,6 +96,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/langfuse/",
     "/vllm/",
     "/mistral/",
+    "/typesafe/",
+    "/nvidia_nim/",
     "/groq/",
     "/voyage/",
     "/cursor/",
@@ -121,6 +124,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
         "/docs/oauth2-redirect",
         "/redoc",
         "/test",
+        "/debug/memory/summary",
     }
 )
 

helm/litellm-helm/templates/_helpers.tpl

@@ -161,3 +161,163 @@ taken before the change, which by that point no longer exists.
 {{- fail (printf "postgresql.image.tag must be pinned to an explicit version when db.deployStandalone is true (got %q). An unpinned tag can start a different PostgreSQL major against the existing data directory, which makes the database unreadable and is not recoverable in place. Crossing a major version requires a dump and restore." $tag) -}}
 {{- end -}}
 {{- end -}}
+
+{{/*
+Environment shared by the proxy container and the opt-in collector sidecar:
+database, pgbouncer, master key, redis, user envVars. Both containers must see
+the same DATABASE_URL and REDIS_* so the sidecar reaches the pod's pgbouncer
+and the same spend transaction buffer.
+*/}}
+{{- define "litellm.proxyEnv" -}}
+- name: HOST
+  value: "{{ .Values.listen | default "0.0.0.0" }}"
+- name: PORT
+  value: {{ .Values.service.port | quote}}
+{{- if .Values.db.deployStandalone }}
+- name: DATABASE_USERNAME
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "litellm.fullname" . }}-dbcredentials
+      key: username
+- name: DATABASE_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "litellm.fullname" . }}-dbcredentials
+      key: password
+- name: DATABASE_HOST
+  value: {{ .Release.Name }}-postgresql
+- name: DATABASE_NAME
+  value: litellm
+{{- else if .Values.db.useExisting }}
+- name: DATABASE_USERNAME
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.db.secret.name }}
+      key: {{ .Values.db.secret.usernameKey }}
+- name: DATABASE_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.db.secret.name }}
+      key: {{ .Values.db.secret.passwordKey }}
+- name: DATABASE_HOST
+  {{- if .Values.db.secret.endpointKey }}
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.db.secret.name }}
+      key: {{ .Values.db.secret.endpointKey }}
+  {{- else }}
+  value: {{ .Values.db.endpoint }}
+  {{- end }}
+- name: DATABASE_NAME
+  value: {{ .Values.db.database }}
+- name: DATABASE_URL
+  value: {{ .Values.db.url | quote }}
+{{- end }}
+{{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }}
+- name: DATABASE_READER_HOST
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.db.secret.name }}
+      key: {{ .Values.db.secret.readReplicaEndpointKey }}
+{{- end }}
+{{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
+- name: DATABASE_URL_READ_REPLICA
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.db.secret.name }}
+      key: {{ .Values.db.secret.readReplicaUrlKey }}
+{{- else if .Values.db.readReplicaUrl }}
+- name: DATABASE_URL_READ_REPLICA
+  value: {{ .Values.db.readReplicaUrl | quote }}
+{{- end }}
+{{- if .Values.db.connectionPool.enabled }}
+- name: LITELLM_PGBOUNCER_ENABLED
+  value: "true"
+- name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+  value: {{ .Values.db.connectionPool.maxDbConnections | quote }}
+- name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+  value: {{ .Values.db.connectionPool.maxClientConn | quote }}
+{{- end }}
+- name: PROXY_MASTER_KEY
+  valueFrom:
+    secretKeyRef:
+      name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }}
+      key: {{ .Values.masterkeySecretKey | default "masterkey" }}
+{{- if .Values.redis.enabled }}
+- name: REDIS_HOST
+  value: {{ include "litellm.redis.serviceName" . }}
+- name: REDIS_PORT
+  value: {{ include "litellm.redis.port" . | quote }}
+- name: REDIS_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "redis.secretName" .Subcharts.redis }}
+      key: {{include "redis.secretPasswordKey" .Subcharts.redis }}
+{{- end }}
+{{- /*
+  Inject LITELLM_LOG only when envVars does not already define it.
+*/}}
+{{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }}
+- name: LITELLM_LOG
+  value: {{ .Values.logLevel | quote }}
+{{- end }}
+{{- if .Values.envVars }}
+{{- range $key, $val := .Values.envVars }}
+- name: {{ $key }}
+  value: {{ $val | quote }}
+{{- end }}
+{{- end }}
+{{- with .Values.extraEnvVars }}
+{{ toYaml . }}
+{{- end }}
+{{- if .Values.migrationJob.enabled }}
+# Schema updates are owned by the dedicated migrations Job; skip
+# the proxy's startup `prisma db push` so N replicas don't race
+# one DB on every rollout. Placed last (after envVars and
+# extraEnvVars) so this override can't be silently shadowed by a
+# user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env
+# semantics — same pattern the migrations Job uses.
+- name: DISABLE_SCHEMA_UPDATE
+  value: "true"
+{{- end }}
+{{- end -}}
+
+{{/*
+Proxy-only metering and metrics env. The collector sidecar serves no HTTP
+traffic, so it gets neither.
+*/}}
+{{- define "litellm.proxyMetricsEnv" -}}
+{{- if .Values.billingMetrics.enabled }}
+{{ include "litellm.billingMetricsEnv" . }}
+{{- end }}
+{{- if .Values.metricsServer.enabled }}
+{{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }}
+{{- fail "metricsServer.port must differ from service.port" }}
+{{- end }}
+- name: PROMETHEUS_METRICS_PORT
+  value: {{ .Values.metricsServer.port | quote }}
+{{- end }}
+{{- end -}}
+
+{{/*
+Directory of the collector's unix socket, shared between the two containers
+through an emptyDir. Empty when the sidecar is off or uses 127.0.0.1 TCP.
+*/}}
+{{- define "litellm.collector.socketDir" -}}
+{{- if and .Values.collector.enabled (hasPrefix "unix://" .Values.collector.address) -}}
+{{- dir (trimPrefix "unix://" .Values.collector.address) -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "litellm.collectorEnv" -}}
+- name: LITELLM_COLLECTOR_ENABLED
+  value: "true"
+- name: LITELLM_COLLECTOR_ADDRESS
+  value: {{ .Values.collector.address | quote }}
+- name: LITELLM_COLLECTOR_BUFFER_SIZE
+  value: {{ .Values.collector.bufferSize | quote }}
+- name: LITELLM_COLLECTOR_ON_UNAVAILABLE
+  value: {{ .Values.collector.onUnavailable | quote }}
+- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS
+  value: {{ .Values.collector.drainTimeoutSeconds | quote }}
+{{- end -}}

helm/litellm-helm/templates/deployment.yaml

@@ -56,118 +56,10 @@ spec:
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           env:
-            - name: HOST
-              value: "{{ .Values.listen | default "0.0.0.0" }}"
-            - name: PORT
-              value: {{ .Values.service.port | quote}}
-            {{- if .Values.db.deployStandalone }}
-            - name: DATABASE_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "litellm.fullname" . }}-dbcredentials
-                  key: username
-            - name: DATABASE_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "litellm.fullname" . }}-dbcredentials
-                  key: password
-            - name: DATABASE_HOST
-              value: {{ .Release.Name }}-postgresql
-            - name: DATABASE_NAME
-              value: litellm
-            {{- else if .Values.db.useExisting }}
-            - name: DATABASE_USERNAME
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.db.secret.name }}
-                  key: {{ .Values.db.secret.usernameKey }}
-            - name: DATABASE_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.db.secret.name }}
-                  key: {{ .Values.db.secret.passwordKey }}
-            - name: DATABASE_HOST
-              {{- if .Values.db.secret.endpointKey }}
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.db.secret.name }}
-                  key: {{ .Values.db.secret.endpointKey }}
-              {{- else }}
-              value: {{ .Values.db.endpoint }}
-              {{- end }}
-            - name: DATABASE_NAME
-              value: {{ .Values.db.database }}
-            - name: DATABASE_URL
-              value: {{ .Values.db.url | quote }}
-            {{- end }}
-            {{- if and .Values.db.useExisting .Values.db.readReplicaUrl .Values.db.secret.readReplicaEndpointKey (not .Values.db.secret.readReplicaUrlKey) }}
-            - name: DATABASE_READER_HOST
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.db.secret.name }}
-                  key: {{ .Values.db.secret.readReplicaEndpointKey }}
-            {{- end }}
-            {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
-            - name: DATABASE_URL_READ_REPLICA
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.db.secret.name }}
-                  key: {{ .Values.db.secret.readReplicaUrlKey }}
-            {{- else if .Values.db.readReplicaUrl }}
-            - name: DATABASE_URL_READ_REPLICA
-              value: {{ .Values.db.readReplicaUrl | quote }}
-            {{- end }}
-            - name: PROXY_MASTER_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: {{ .Values.masterkeySecretName | default (printf "%s-masterkey" (include "litellm.fullname" .)) }}
-                  key: {{ .Values.masterkeySecretKey | default "masterkey" }}
-            {{- if .Values.redis.enabled }}
-            - name: REDIS_HOST
-              value: {{ include "litellm.redis.serviceName" . }}
-            - name: REDIS_PORT
-              value: {{ include "litellm.redis.port" . | quote }}
-            - name: REDIS_PASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: {{ include "redis.secretName" .Subcharts.redis }}
-                  key: {{include "redis.secretPasswordKey" .Subcharts.redis }}
-            {{- end }}
-            {{- /*
-              Inject LITELLM_LOG only when envVars does not already define it.
-            */}}
-            {{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }}
-            - name: LITELLM_LOG
-              value: {{ .Values.logLevel | quote }}
-            {{- end }}
-            {{- if .Values.envVars }}
-            {{- range $key, $val := .Values.envVars }}
-            - name: {{ $key }}
-              value: {{ $val | quote }}
-            {{- end }}
-            {{- end }}
-            {{- with .Values.extraEnvVars }}
-              {{- toYaml . | nindent 12 }}
-            {{- end }}
-            {{- if .Values.billingMetrics.enabled }}
-            {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
-            {{- end }}
-            {{- if .Values.metricsServer.enabled }}
-            {{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }}
-            {{- fail "metricsServer.port must differ from service.port" }}
-            {{- end }}
-            - name: PROMETHEUS_METRICS_PORT
-              value: {{ .Values.metricsServer.port | quote }}
-            {{- end }}
-            {{- if .Values.migrationJob.enabled }}
-            # Schema updates are owned by the dedicated migrations Job; skip
-            # the proxy's startup `prisma db push` so N replicas don't race
-            # one DB on every rollout. Placed last (after envVars and
-            # extraEnvVars) so this override can't be silently shadowed by a
-            # user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env
-            # semantics — same pattern the migrations Job uses.
-            - name: DISABLE_SCHEMA_UPDATE
-              value: "true"
+            {{- include "litellm.proxyEnv" . | nindent 12 }}
+            {{- include "litellm.proxyMetricsEnv" . | nindent 12 }}
+            {{- if .Values.collector.enabled }}
+            {{- include "litellm.collectorEnv" . | nindent 12 }}
             {{- end }}
           envFrom:
           {{- range .Values.environmentSecrets }}
@@ -245,6 +137,10 @@ spec:
           {{- if .Values.billingMetrics.enabled }}
             {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
           {{- end }}
+          {{- if include "litellm.collector.socketDir" . }}
+            - name: collector-socket
+              mountPath: {{ include "litellm.collector.socketDir" . }}
+          {{- end }}
           {{- with .Values.volumeMounts }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
@@ -252,6 +148,53 @@ spec:
           lifecycle:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+      {{- if .Values.collector.enabled }}
+        - name: {{ include "litellm.name" . }}-collector
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: {{ toYaml .Values.collector.command | nindent 12 }}
+          env:
+            {{- include "litellm.proxyEnv" . | nindent 12 }}
+            {{- include "litellm.collectorEnv" . | nindent 12 }}
+            - name: LITELLM_JOB_ROLE
+              value: collector
+            {{- if not (hasKey (default dict .Values.envVars) "CONFIG_FILE_PATH") }}
+            - name: CONFIG_FILE_PATH
+              value: /etc/litellm/config.yaml
+            {{- end }}
+          envFrom:
+          {{- range .Values.environmentSecrets }}
+            - secretRef:
+                name: {{ . }}
+          {{- end }}
+          {{- range .Values.environmentConfigMaps }}
+            - configMapRef:
+                name: {{ . }}
+          {{- end }}
+          resources:
+            {{- toYaml .Values.collector.resources | nindent 12 }}
+          volumeMounts:
+            - name: litellm-config
+              mountPath: /etc/litellm/config.yaml
+              subPath: config.yaml
+          {{- if include "litellm.collector.socketDir" . }}
+            - name: collector-socket
+              mountPath: {{ include "litellm.collector.socketDir" . }}
+          {{- end }}
+          {{ if .Values.securityContext.readOnlyRootFilesystem }}
+            - name: tmp
+              mountPath: /tmp
+            - name: cache
+              mountPath: /.cache
+            - name: npm
+              mountPath: /.npm
+          {{- end }}
+          {{- with .Values.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+      {{- end }}
       {{- with .Values.extraContainers }}
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
@@ -280,6 +223,11 @@ spec:
         {{- if .Values.billingMetrics.enabled }}
         {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
         {{- end }}
+        {{- if include "litellm.collector.socketDir" . }}
+        - name: collector-socket
+          emptyDir:
+            sizeLimit: 1Mi
+        {{- end }}
       {{- with .Values.volumes }}
         {{- toYaml . | nindent 8 }}
       {{- end }}

helm/litellm-helm/templates/hpa.yaml

@@ -18,6 +18,15 @@ spec:
   {{- end }}
   metrics:
     {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
+    {{- if and .Values.collector.enabled .Values.collector.scaleOnProxyContainerCpu }}
+    - type: ContainerResource
+      containerResource:
+        name: cpu
+        container: {{ include "litellm.name" . }}
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
+    {{- else }}
     - type: Resource
       resource:
         name: cpu
@@ -25,6 +34,7 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
     {{- end }}
+    {{- end }}
     {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
     - type: Resource
       resource:
@@ -33,4 +43,22 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
     {{- end }}
+    {{- with .Values.autoscaling.targetRequestsPerSecond }}
+    - type: Pods
+      pods:
+        metric:
+          name: litellm_requests_per_second
+        target:
+          type: AverageValue
+          averageValue: {{ toJson . | trimAll "\"" | quote }}
+    {{- end }}
+    {{- with .Values.autoscaling.targetTokensPerSecond }}
+    - type: Pods
+      pods:
+        metric:
+          name: litellm_tokens_per_second
+        target:
+          type: AverageValue
+          averageValue: {{ toJson . | trimAll "\"" | quote }}
+    {{- end }}
 {{- end }}

helm/litellm-helm/templates/keda.yaml

@@ -23,6 +23,27 @@ spec:
   triggers:
 {{- with .Values.keda.triggers }}
   {{- toYaml . | nindent 2 }}
+{{- end }}
+{{- $prom := .Values.keda.prometheus }}
+{{- if or $prom.requestsPerSecond $prom.tokensPerSecond }}
+{{- if not $prom.serverAddress }}
+{{- fail "keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set" }}
+{{- end }}
+{{- $selector := printf "namespace=%q,job=%q" .Release.Namespace (printf "%s%s" (include "litellm.fullname" .) (ternary "-metrics" "" .Values.metricsServer.enabled)) }}
+{{- with $prom.requestsPerSecond }}
+  - type: prometheus
+    metadata:
+      serverAddress: {{ $prom.serverAddress | quote }}
+      threshold: {{ toJson . | trimAll "\"" | quote }}
+      query: {{ printf "sum(rate(litellm_proxy_total_requests_metric_total{%s}[1m]))" $selector | quote }}
+{{- end }}
+{{- with $prom.tokensPerSecond }}
+  - type: prometheus
+    metadata:
+      serverAddress: {{ $prom.serverAddress | quote }}
+      threshold: {{ toJson . | trimAll "\"" | quote }}
+      query: {{ printf "sum(rate(litellm_total_tokens_metric_total{%s}[1m]))" $selector | quote }}
+{{- end }}
 {{- end }}
   advanced:
     restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }}

helm/litellm-helm/tests/collector_tests.yaml

@@ -0,0 +1,272 @@
+suite: test collector sidecar
+templates:
+  - deployment.yaml
+  - hpa.yaml
+  - configmap-litellm.yaml
+tests:
+  - it: should run the proxy alone with no collector env by default
+    template: deployment.yaml
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.containers
+          count: 1
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_ENABLED
+            value: "true"
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: collector-socket
+          any: true
+
+  - it: should add the sidecar on the same image and point both containers at the unix socket
+    template: deployment.yaml
+    set:
+      image.tag: test
+      db.connectionPool.enabled: true
+      collector.enabled: true
+      collector.resources:
+        requests:
+          cpu: 500m
+          memory: 1Gi
+        limits:
+          cpu: "1"
+          memory: 2Gi
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.containers
+          count: 2
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: litellm-collector
+      - equal:
+          path: spec.template.spec.containers[1].image
+          value: ghcr.io/berriai/litellm:test
+      - equal:
+          path: spec.template.spec.containers[1].command
+          value: [python, -m, litellm.proxy.collector]
+      - equal:
+          path: spec.template.spec.containers[1].resources.requests.cpu
+          value: 500m
+      - equal:
+          path: spec.template.spec.containers[1].resources.limits.memory
+          value: 2Gi
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_ENABLED
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_ADDRESS
+            value: unix:///var/run/litellm/collector.sock
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_BUFFER_SIZE
+            value: "1000"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_ON_UNAVAILABLE
+            value: fallback
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_JOB_ROLE
+            value: collector
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_JOB_ROLE
+            value: collector
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_COLLECTOR_ADDRESS
+            value: unix:///var/run/litellm/collector.sock
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: CONFIG_FILE_PATH
+            value: /etc/litellm/config.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: DATABASE_HOST
+            value: RELEASE-NAME-postgresql
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: DATABASE_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: RELEASE-NAME-litellm-dbcredentials
+                key: password
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+            value: "20"
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: collector-socket
+            mountPath: /var/run/litellm
+      - contains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: collector-socket
+            mountPath: /var/run/litellm
+      - contains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: litellm-config
+            mountPath: /etc/litellm/config.yaml
+            subPath: config.yaml
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: collector-socket
+            emptyDir:
+              sizeLimit: 1Mi
+
+  - it: should skip the socket volume and pass the policy through on tcp transport
+    template: deployment.yaml
+    set:
+      collector.enabled: true
+      collector.address: tcp://127.0.0.1:4100
+      collector.onUnavailable: drop
+      collector.bufferSize: 50
+      envVars:
+        CONFIG_FILE_PATH: /custom/config.yaml
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.containers
+          count: 2
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: CONFIG_FILE_PATH
+            value: /etc/litellm/config.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: CONFIG_FILE_PATH
+            value: /custom/config.yaml
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_ADDRESS
+            value: tcp://127.0.0.1:4100
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_ON_UNAVAILABLE
+            value: drop
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_BUFFER_SIZE
+            value: "50"
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: collector-socket
+          any: true
+
+  - it: should keep metrics and billing env on the proxy container only
+    template: deployment.yaml
+    set:
+      collector.enabled: true
+      metricsServer.enabled: true
+      metricsServer.port: 9090
+      billingMetrics.enabled: true
+      billingMetrics.endpoint: https://metering.example.com
+      billingMetrics.secretName: billing-mtls
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PROMETHEUS_METRICS_PORT
+            value: "9090"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://metering.example.com
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: PROMETHEUS_METRICS_PORT
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: billing-metrics-mtls
+          any: true
+
+  - it: should give the sidecar the same scratch mounts as the proxy on a read-only root
+    template: deployment.yaml
+    set:
+      collector.enabled: true
+      securityContext.readOnlyRootFilesystem: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: npm
+            mountPath: /.npm
+      - contains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: cache
+            mountPath: /.cache
+      - contains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: tmp
+            mountPath: /tmp
+
+  - it: should keep the pod-wide cpu metric unless asked to scale on the proxy container
+    template: hpa.yaml
+    set:
+      autoscaling.enabled: true
+      collector.enabled: true
+    asserts:
+      - equal: { path: "spec.metrics[0].type", value: Resource }
+      - equal: { path: "spec.metrics[0].resource.name", value: cpu }
+
+  - it: should scale on the proxy container's cpu only when opted in
+    template: hpa.yaml
+    set:
+      autoscaling.enabled: true
+      collector.enabled: true
+      collector.scaleOnProxyContainerCpu: true
+    asserts:
+      - equal: { path: "spec.metrics[0].type", value: ContainerResource }
+      - equal: { path: "spec.metrics[0].containerResource.name", value: cpu }
+      - equal: { path: "spec.metrics[0].containerResource.container", value: litellm }
+      - equal: { path: "spec.metrics[0].containerResource.target.averageUtilization", value: 60 }
+      - isNull: { path: "spec.metrics[0].resource" }
+
+  - it: should not switch to the container metric while the sidecar is off
+    template: hpa.yaml
+    set:
+      autoscaling.enabled: true
+      collector.scaleOnProxyContainerCpu: true
+    asserts:
+      - equal: { path: "spec.metrics[0].type", value: Resource }

helm/litellm-helm/tests/connection_pool_tests.yaml

@@ -0,0 +1,112 @@
+suite: test in-container connection pool
+templates:
+  - deployment.yaml
+  - configmap-litellm.yaml
+tests:
+  - it: should not emit pgbouncer env vars by default
+    template: deployment.yaml
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+            value: "20"
+
+  - it: should enable the pool with the default sizing when connectionPool.enabled is set
+    template: deployment.yaml
+    set:
+      db.connectionPool.enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+            value: "20"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+            value: "1000"
+
+  - it: should pass custom sizing through as strings next to the worker count
+    template: deployment.yaml
+    set:
+      numWorkers: 4
+      db.connectionPool.enabled: true
+      db.connectionPool.maxDbConnections: 8
+      db.connectionPool.maxClientConn: 400
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+            value: "8"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+            value: "400"
+      - contains:
+          path: spec.template.spec.containers[0].args
+          content: "4"
+
+  - it: should give the collector sidecar the same pool env as the proxy container
+    template: deployment.yaml
+    set:
+      collector.enabled: true
+      db.connectionPool.enabled: true
+      db.connectionPool.maxDbConnections: 8
+      db.connectionPool.maxClientConn: 400
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: litellm-collector
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+            value: "8"
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+            value: "400"
+
+  - it: should give the collector sidecar no pool env when the pool is off
+    template: deployment.yaml
+    set:
+      collector.enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: litellm-collector
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+          any: true

helm/litellm-helm/tests/hpa_tests.yaml

@@ -61,6 +61,84 @@ tests:
       - equal: { path: "spec.metrics[1].resource.name", value: memory }
       - equal: { path: "spec.metrics[1].resource.target.averageUtilization", value: 80 }
 
+  - it: "renders no workload metrics by default"
+    set:
+      autoscaling.enabled: true
+      autoscaling.targetMemoryUtilizationPercentage: 80
+    asserts:
+      - lengthEqual: { path: spec.metrics, count: 2 }
+      - notContains: { path: spec.metrics, content: { type: Pods }, any: true }
+
+  - it: "adds a requests-per-second Pods metric after the cpu metric"
+    set:
+      autoscaling.enabled: true
+      autoscaling.targetRequestsPerSecond: 90
+    asserts:
+      - lengthEqual: { path: spec.metrics, count: 2 }
+      - equal: { path: "spec.metrics[0].resource.name", value: cpu }
+      - equal:
+          path: "spec.metrics[1]"
+          value:
+            type: Pods
+            pods:
+              metric: { name: litellm_requests_per_second }
+              target: { type: AverageValue, averageValue: "90" }
+
+  - it: "adds a tokens-per-second Pods metric on its own"
+    set:
+      autoscaling.enabled: true
+      autoscaling.targetTokensPerSecond: 6M
+    asserts:
+      - lengthEqual: { path: spec.metrics, count: 2 }
+      - equal:
+          path: "spec.metrics[1]"
+          value:
+            type: Pods
+            pods:
+              metric: { name: litellm_tokens_per_second }
+              target: { type: AverageValue, averageValue: "6M" }
+      - notContains:
+          path: spec.metrics
+          content: { type: Pods, pods: { metric: { name: litellm_requests_per_second } } }
+          any: true
+
+  - it: "renders requests, tokens, cpu and memory metrics together"
+    set:
+      autoscaling.enabled: true
+      autoscaling.targetMemoryUtilizationPercentage: 80
+      autoscaling.targetRequestsPerSecond: 90
+      autoscaling.targetTokensPerSecond: 6000000
+    asserts:
+      - lengthEqual: { path: spec.metrics, count: 4 }
+      - equal: { path: "spec.metrics[0].resource.name", value: cpu }
+      - equal: { path: "spec.metrics[1].resource.name", value: memory }
+      - equal: { path: "spec.metrics[2].pods.metric.name", value: litellm_requests_per_second }
+      - equal: { path: "spec.metrics[2].pods.target.averageValue", value: "90" }
+      - equal: { path: "spec.metrics[3].pods.metric.name", value: litellm_tokens_per_second }
+      - equal: { path: "spec.metrics[3].pods.target.averageValue", value: "6000000" }
+
+  - it: "scales on workload metrics alone when the cpu target is cleared"
+    set:
+      autoscaling.enabled: true
+      autoscaling.targetCPUUtilizationPercentage: null
+      autoscaling.targetRequestsPerSecond: 90
+      autoscaling.targetTokensPerSecond: 6000000
+    asserts:
+      - lengthEqual: { path: spec.metrics, count: 2 }
+      - notContains: { path: spec.metrics, content: { type: Resource }, any: true }
+      - equal: { path: "spec.metrics[0].pods.metric.name", value: litellm_requests_per_second }
+      - equal: { path: "spec.metrics[1].pods.metric.name", value: litellm_tokens_per_second }
+      - notMatchRegexRaw: { pattern: per_minute }
+
+  - it: "ignores the per-minute keys, which the chart never shipped"
+    set:
+      autoscaling.enabled: true
+      autoscaling.targetRequestsPerMinute: 5400
+      autoscaling.targetTokensPerMinute: 360000000
+    asserts:
+      - lengthEqual: { path: spec.metrics, count: 1 }
+      - notContains: { path: spec.metrics, content: { type: Pods }, any: true }
+
   - it: "renders no hpa when autoscaling is disabled"
     asserts:
       - hasDocuments: { count: 0 }

helm/litellm-helm/tests/keda_tests.yaml

@@ -0,0 +1,106 @@
+suite: "keda"
+templates:
+  - keda.yaml
+release:
+  name: rel
+  namespace: llm
+tests:
+  - it: "renders no scaled object by default"
+    asserts:
+      - hasDocuments: { count: 0 }
+
+  - it: "passes user triggers through and adds no prometheus triggers by default"
+    set:
+      keda.enabled: true
+      keda.triggers:
+        - type: cpu
+          metricType: Utilization
+          metadata: { value: "60" }
+    asserts:
+      - isKind: { of: ScaledObject }
+      - equal:
+          path: spec.triggers
+          value:
+            - type: cpu
+              metricType: Utilization
+              metadata: { value: "60" }
+
+  - it: "scales on release-wide requests per second divided by the per-replica target"
+    set:
+      keda.enabled: true
+      keda.prometheus.serverAddress: http://prometheus-operated.monitoring.svc:9090
+      keda.prometheus.requestsPerSecond: 90
+    asserts:
+      - lengthEqual: { path: spec.triggers, count: 1 }
+      - equal:
+          path: "spec.triggers[0]"
+          value:
+            type: prometheus
+            metadata:
+              serverAddress: http://prometheus-operated.monitoring.svc:9090
+              threshold: "90"
+              query: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm"}[1m]))
+
+  - it: "scales on tokens per second on its own"
+    set:
+      keda.enabled: true
+      keda.prometheus.serverAddress: http://prom:9090
+      keda.prometheus.tokensPerSecond: 6000000
+    asserts:
+      - lengthEqual: { path: spec.triggers, count: 1 }
+      - equal: { path: "spec.triggers[0].type", value: prometheus }
+      - equal: { path: "spec.triggers[0].metadata.threshold", value: "6000000" }
+      - equal:
+          path: "spec.triggers[0].metadata.query"
+          value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm"}[1m]))
+
+  - it: "appends requests and tokens triggers after user triggers and selects the metrics service job"
+    set:
+      keda.enabled: true
+      metricsServer.enabled: true
+      keda.triggers:
+        - type: cpu
+          metricType: Utilization
+          metadata: { value: "60" }
+      keda.prometheus.serverAddress: http://prom:9090
+      keda.prometheus.requestsPerSecond: 90
+      keda.prometheus.tokensPerSecond: 6000000
+    asserts:
+      - lengthEqual: { path: spec.triggers, count: 3 }
+      - equal: { path: "spec.triggers[0].type", value: cpu }
+      - equal: { path: "spec.triggers[1].metadata.threshold", value: "90" }
+      - equal:
+          path: "spec.triggers[1].metadata.query"
+          value: sum(rate(litellm_proxy_total_requests_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m]))
+      - equal: { path: "spec.triggers[2].metadata.threshold", value: "6000000" }
+      - equal:
+          path: "spec.triggers[2].metadata.query"
+          value: sum(rate(litellm_total_tokens_metric_total{namespace="llm",job="rel-litellm-metrics"}[1m]))
+      - notMatchRegexRaw: { pattern: "\\* *60|per_minute|PerMinute" }
+
+  - it: "ignores the per-minute keys, which the chart never shipped"
+    set:
+      keda.enabled: true
+      keda.prometheus.serverAddress: http://prom:9090
+      keda.prometheus.requestsPerMinute: 5400
+      keda.prometheus.tokensPerMinute: 360000000
+    asserts:
+      - isKind: { of: ScaledObject }
+      - isNullOrEmpty: { path: spec.triggers }
+
+  - it: "refuses a workload target without a prometheus server address"
+    set:
+      keda.enabled: true
+      keda.prometheus.requestsPerSecond: 90
+    asserts:
+      - failedTemplate:
+          errorMessage: keda.prometheus.serverAddress is required when keda.prometheus.requestsPerSecond or tokensPerSecond is set
+
+  - it: "yields to the hpa when both autoscalers are enabled"
+    set:
+      autoscaling.enabled: true
+      keda.enabled: true
+      keda.prometheus.serverAddress: http://prom:9090
+      keda.prometheus.requestsPerSecond: 90
+    asserts:
+      - hasDocuments: { count: 0 }

helm/litellm-helm/values.yaml

@@ -190,6 +190,48 @@ metricsServer:
   enabled: false
   port: 4001
 
+# Opt-in sidecar that runs the post-response spend pipeline (cost calculation,
+# spend logs, spend counters, budget reservation reconciliation) so the proxy's
+# uvicorn workers only serialise a compact typed event and go back to serving
+# inference. Same image and tag as the proxy, second container in the same pod,
+# fed over loopback (a unix socket on a shared emptyDir, or 127.0.0.1 TCP). It
+# reuses the pod's in-container pgbouncer (db.connectionPool) and the same Redis
+# spend transaction buffer, so the per-pod DB connection budget is unchanged.
+# Delivery is at-most-once inside the pod: events already handed to the sidecar
+# are lost if it crashes before writing them; events the workers could not hand
+# over follow onUnavailable. Both containers drain on SIGTERM within
+# terminationGracePeriodSeconds
+collector:
+  enabled: false
+  # unix:///<dir>/<file>.sock (the <dir> becomes a shared emptyDir) or tcp://127.0.0.1:<port>
+  address: unix:///var/run/litellm/collector.sock
+  # Events each uvicorn worker holds in memory while the sidecar is slow or restarting
+  bufferSize: 1000
+  # fallback: run the pipeline in the worker when the sidecar is unreachable or the
+  # buffer is full (spend stays exact, that request costs proxy CPU again)
+  # drop: count and discard the event instead (spend under-reports)
+  onUnavailable: fallback
+  # How long the workers keep pushing buffered events on shutdown, and how long the
+  # sidecar keeps serving its open connections after SIGTERM
+  drainTimeoutSeconds: 10
+  command:
+    - python
+    - -m
+    - litellm.proxy.collector
+  # Sized independently of the proxy container; the pipeline is CPU bound
+  resources: {}
+  # requests:
+  #   cpu: 500m
+  #   memory: 1Gi
+  # limits:
+  #   cpu: "1"
+  #   memory: 2Gi
+  # When autoscaling.enabled, swap the pod-wide cpu Resource metric for an
+  # autoscaling/v2 ContainerResource metric on the proxy container only, so the
+  # sidecar's CPU never scales inference replicas. Needs Kubernetes 1.30+ (or the
+  # HPAContainerMetrics feature gate on 1.27 to 1.29)
+  scaleOnProxyContainerCpu: false
+
 resources:
   {}
   # Unset by default so the chart installs on small clusters such as Minikube, and so an
@@ -222,6 +264,25 @@ autoscaling:
   # Memory is a floor to provision under 'resources', not a signal to scale on.
   # targetMemoryUtilizationPercentage: 80
   # behavior: {}
+  # Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics
+  # named `litellm_requests_per_second` and `litellm_tokens_per_second` with an
+  # AverageValue target, alongside whichever resource targets are set (the HPA
+  # follows the metric asking for the most replicas). A Prometheus Adapter must
+  # serve those two names on custom.metrics.k8s.io from the proxy's counters,
+  # grouped by the scrape target's `pod` label (enable serviceMonitor below so
+  # every pod is scraped on its own):
+  #   litellm_requests_per_second:
+  #     sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+  #   litellm_tokens_per_second:
+  #     sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+  # rate() over [1m] is already per second, so no `* 60`. How fast the HPA
+  # reacts is set by that window, the scrape interval and the HPA sync period
+  # (15s by default), not by the unit: keep serviceMonitor.interval at 15s or
+  # faster so a 1m window holds at least 4 samples. averageValue takes SI
+  # suffixes, so "6M" is six million tokens per second per pod. Tokens are
+  # counted when a response completes, so TPS trails long streams.
+  targetRequestsPerSecond: ""
+  targetTokensPerSecond: ""
 
 # Autoscaling with keda is mutually exclusive with hpa
 keda:
@@ -243,6 +304,23 @@ keda:
   #     metricName: http_requests_total
   #     threshold: '100'
   #     query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
+  # First-class Prometheus triggers on the proxy's own request and token
+  # counters, appended to `triggers`. Each target is the per-second load one
+  # replica should carry: KEDA divides the release-wide
+  # `sum(rate(<counter>[1m]))` by it to pick the replica count. Thresholds
+  # are plain numbers (KEDA parses them as floats, no SI suffixes). The
+  # queries select samples by the release namespace and the `job` label the
+  # chart's ServiceMonitor produces (the metrics Service name), so enable
+  # serviceMonitor below together with metricsServer: the http port serves
+  # /metrics/ behind virtual-key auth and answers an unauthenticated scrape
+  # with 401. Reaction time comes from the [1m] window, the scrape interval
+  # and pollingInterval above, so keep both at 15s or faster. Tokens are
+  # counted at completion, so TPS trails long streams. serverAddress is
+  # required once either target is set.
+  prometheus:
+    serverAddress: ""
+    requestsPerSecond: ""
+    tokensPerSecond: ""
   behavior: {}
   # scaleDown:
   #   stabilizationWindowSeconds: 300
@@ -319,6 +397,20 @@ db:
   # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime).
   readReplicaUrl: ""
 
+  # In-container connection pool (PgBouncer, transaction mode) shared by every
+  # worker in the pod. Without it each --num_workers worker opens its own
+  # connection_limit connections to Postgres, so a pod's footprint against the
+  # database's connection ceiling is workers x connection_limit and grows with
+  # every replica. With it, the pod holds at most maxDbConnections upstream
+  # connections no matter how many workers run; the workers connect to the pool
+  # over loopback, with no extra network hop. Migrations still go straight to
+  # Postgres. Starting profile for numWorkers: 4 is maxDbConnections: 20, so
+  # a database with a 5000-connection ceiling fits roughly 200 replicas.
+  connectionPool:
+    enabled: false
+    maxDbConnections: 20
+    maxClientConn: 1000
+
   # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
   #  The Stackgres Operator must already be installed within the target
   #  Kubernetes cluster.

helm/litellm/templates/_helpers.tpl

@@ -257,6 +257,14 @@ IAM_TOKEN_DB_AUTH / AZURE_POSTGRESQL_AUTH toggle that only the writer sets.
 - name: DATABASE_SCHEMA
   value: {{ .schema | quote }}
 {{- end }}
+{{- if .sslMode }}
+- name: DATABASE_SSLMODE
+  value: {{ .sslMode | quote }}
+{{- end }}
+{{- if .sslRootCert }}
+- name: DATABASE_SSLROOTCERT
+  value: {{ .sslRootCert | quote }}
+{{- end }}
 {{- if and .useIAMAuth .useAzureEntraAuth }}
 {{- fail "database.writer.useIAMAuth and database.writer.useAzureEntraAuth are mutually exclusive: the database password can only come from one token source" }}
 {{- end }}
@@ -360,6 +368,20 @@ harmless no-op for the Job and authoritative for the app pods.
 {{- end }}
 {{- end -}}
 
+{{/*
+In-container PgBouncer env for the gateway container. Under IAM or Entra auth the pooler mints and renews the database token itself.
+*/}}
+{{- define "litellm.connectionPoolEnv" -}}
+{{- with .Values.database.connectionPool -}}
+- name: LITELLM_PGBOUNCER_ENABLED
+  value: "true"
+- name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+  value: {{ required "database.connectionPool.maxDbConnections is required when the pool is enabled" .maxDbConnections | quote }}
+- name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+  value: {{ required "database.connectionPool.maxClientConn is required when the pool is enabled" .maxClientConn | quote }}
+{{- end }}
+{{- end -}}
+
 {{/*
 PodDisruptionBudget shared by gateway, backend, and ui.
 
@@ -443,3 +465,34 @@ ImplementationSpecific
 {{- end -}}
 
 {{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}}
+
+{{/*
+Directory of the collector's unix socket, shared by the gateway and
+collector containers through an emptyDir. Empty when the sidecar is off
+or gateway.collector.address is a tcp://127.0.0.1:<port> address.
+*/}}
+{{- define "litellm.gateway.collectorSocketDir" -}}
+{{- if and .Values.gateway.collector.enabled (hasPrefix "unix://" .Values.gateway.collector.address) -}}
+{{- dir (trimPrefix "unix://" .Values.gateway.collector.address) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+LITELLM_COLLECTOR_* env shared by the producer (gateway container) and the
+consumer (collector container), so both agree on the transport and the
+shutdown drain window.
+*/}}
+{{- define "litellm.gateway.collectorEnv" -}}
+{{- with .Values.gateway.collector }}
+- name: LITELLM_COLLECTOR_ENABLED
+  value: "true"
+- name: LITELLM_COLLECTOR_ADDRESS
+  value: {{ .address | quote }}
+- name: LITELLM_COLLECTOR_BUFFER_SIZE
+  value: {{ .bufferSize | quote }}
+- name: LITELLM_COLLECTOR_ON_UNAVAILABLE
+  value: {{ .onUnavailable | quote }}
+- name: LITELLM_COLLECTOR_DRAIN_TIMEOUT_SECONDS
+  value: {{ .drainTimeoutSeconds | quote }}
+{{- end }}
+{{- end -}}

helm/litellm/templates/gateway/deployment.yaml

@@ -61,6 +61,9 @@ spec:
             - name: NUM_WORKERS
               value: {{ .Values.gateway.numWorkers | quote }}
             {{- end }}
+            {{- if .Values.database.connectionPool.enabled }}
+            {{- include "litellm.connectionPoolEnv" $ | nindent 12 }}
+            {{- end }}
             {{- if .Values.billingMetrics.enabled }}
             {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
             {{- end }}
@@ -71,8 +74,11 @@ spec:
             - name: PROMETHEUS_MULTIPROC_DIR
               value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
             {{- end }}
+            {{- if .Values.gateway.collector.enabled }}
+            {{- include "litellm.gateway.collectorEnv" . | nindent 12 }}
+            {{- end }}
           {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }}
+          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
@@ -83,6 +89,10 @@ spec:
             - name: prometheus-multiproc
               mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
             {{- end }}
+            {{- if include "litellm.gateway.collectorSocketDir" . }}
+            - name: collector-socket
+              mountPath: {{ include "litellm.gateway.collectorSocketDir" . }}
+            {{- end }}
             {{- if .Values.billingMetrics.enabled }}
             {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
             {{- end }}
@@ -142,10 +152,53 @@ spec:
           resources:
             {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }}
         {{- end }}
+        {{- if .Values.gateway.collector.enabled }}
+        - name: collector
+          image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
+          {{- with .Values.gateway.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          command:
+            - python
+            - -m
+            - litellm.proxy.collector
+          env:
+            {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.gateway) | nindent 12 }}
+            {{- if .Values.gateway.config.create }}
+            - name: CONFIG_FILE_PATH
+              value: /app/config/config.yaml
+            {{- end }}
+            {{- if .Values.database.connectionPool.enabled }}
+            {{- include "litellm.connectionPoolEnv" $ | nindent 12 }}
+            {{- end }}
+            {{- include "litellm.gateway.collectorEnv" . | nindent 12 }}
+            - name: LITELLM_JOB_ROLE
+              value: collector
+          {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
+          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts (include "litellm.gateway.collectorSocketDir" .) }}
+          volumeMounts:
+            {{- if .Values.gateway.config.create }}
+            - name: gateway-config
+              mountPath: /app/config/config.yaml
+              subPath: config.yaml
+            {{- end }}
+            {{- if include "litellm.gateway.collectorSocketDir" . }}
+            - name: collector-socket
+              mountPath: {{ include "litellm.gateway.collectorSocketDir" . }}
+            {{- end }}
+            {{- with .Values.gateway.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
+          {{- end }}
+          resources:
+            {{- toYaml .Values.gateway.collector.resources | nindent 12 }}
+        {{- end }}
       {{- with .Values.gateway.extraContainers }}
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
-      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }}
+      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
@@ -156,6 +209,11 @@ spec:
         - name: prometheus-multiproc
           emptyDir: {}
         {{- end }}
+        {{- if include "litellm.gateway.collectorSocketDir" . }}
+        - name: collector-socket
+          emptyDir:
+            sizeLimit: 1Mi
+        {{- end }}
         {{- if .Values.billingMetrics.enabled }}
         {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
         {{- end }}

helm/litellm/templates/gateway/hpa.yaml

@@ -15,6 +15,15 @@ spec:
   maxReplicas: {{ .Values.gateway.hpa.maxReplicas }}
   metrics:
     {{- if .Values.gateway.hpa.targetCPUUtilizationPercentage }}
+    {{- if and .Values.gateway.collector.enabled .Values.gateway.collector.scaleOnGatewayContainerCpu }}
+    - type: ContainerResource
+      containerResource:
+        name: cpu
+        container: gateway
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
+    {{- else }}
     - type: Resource
       resource:
         name: cpu
@@ -22,6 +31,7 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.gateway.hpa.targetCPUUtilizationPercentage }}
     {{- end }}
+    {{- end }}
     {{- if .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
     - type: Resource
       resource:
@@ -30,6 +40,24 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
     {{- end }}
+    {{- with .Values.gateway.hpa.targetRequestsPerSecond }}
+    - type: Pods
+      pods:
+        metric:
+          name: litellm_requests_per_second
+        target:
+          type: AverageValue
+          averageValue: {{ toJson . | trimAll "\"" | quote }}
+    {{- end }}
+    {{- with .Values.gateway.hpa.targetTokensPerSecond }}
+    - type: Pods
+      pods:
+        metric:
+          name: litellm_tokens_per_second
+        target:
+          type: AverageValue
+          averageValue: {{ toJson . | trimAll "\"" | quote }}
+    {{- end }}
   {{- with .Values.gateway.hpa.behavior }}
   behavior:
     {{- toYaml . | nindent 4 }}

helm/litellm/templates/gateway/servicemonitor.yaml

@@ -0,0 +1,28 @@
+{{- if and .Values.gateway.enabled .Values.gateway.serviceMonitor.enabled }}
+{{- if not .Values.gateway.metricsServer.enabled }}
+{{- fail "gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled: the http port serves /metrics/ behind virtual-key auth, so an unauthenticated scrape gets 401" }}
+{{- end }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "litellm.gateway.fullname" . }}
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: gateway
+    {{- with .Values.gateway.serviceMonitor.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "litellm.gateway.selectorLabels" . | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace | quote }}
+  endpoints:
+    - port: metrics
+      path: /metrics/
+      interval: {{ .Values.gateway.serviceMonitor.interval }}
+      scrapeTimeout: {{ .Values.gateway.serviceMonitor.scrapeTimeout }}
+      scheme: http
+{{- end }}

helm/litellm/templates/ingress.yaml

@@ -89,7 +89,7 @@
   at "/" Prefix would swallow the whole backend management API) instead of
   adding to it.
 */}}
-{{- $builtinPathKeys := list "/test|Exact" "/|Prefix" -}}
+{{- $builtinPathKeys := list "/test|Exact" "/debug/memory/summary|Exact" "/|Prefix" -}}
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -129,6 +129,8 @@ spec:
           # --- Gateway data plane ---
           # Exact /test only (see the $gatewayPrefixes comment above);
           # /test/* MCP management endpoints fall to the backend catch-all.
+          # Exact /debug/memory/summary reads a serving worker's RSS (the e2e memory
+          # gate); the rest of /debug/* stays on the backend.
           - path: /test
             pathType: Exact
             backend:
@@ -136,6 +138,13 @@ spec:
                 name: {{ $gatewayName }}
                 port:
                   number: {{ $gatewayPort }}
+          - path: /debug/memory/summary
+            pathType: Exact
+            backend:
+              service:
+                name: {{ $gatewayName }}
+                port:
+                  number: {{ $gatewayPort }}
           {{- range $gatewayPrefixes }}
           {{- $pathType := include "litellm.ingress.pathType" (dict "controller" $controller "path" . "pathType" "Prefix") }}
           {{- $builtinPathKeys = append $builtinPathKeys (printf "%s|%s" . $pathType) }}

helm/litellm/tests/collector_tests.yaml

@@ -0,0 +1,216 @@
+suite: test gateway collector sidecar
+templates:
+  - gateway/configmap.yaml
+  - gateway/deployment.yaml
+  - gateway/hpa.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: adds no sidecar, env, volume or container metric when the collector is off
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.containers
+          count: 1
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_ENABLED
+            value: "true"
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: collector-socket
+          any: true
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.metrics[0].type
+          value: Resource
+        template: gateway/hpa.yaml
+
+  - it: runs the collector as a sidecar sharing env, config, the pod pool and a unix socket emptyDir, and scales on the gateway container only
+    set:
+      gateway.collector.enabled: true
+      gateway.collector.bufferSize: 250
+      gateway.collector.onUnavailable: drop
+      gateway.image.tag: v1.102.0
+      gateway.numWorkers: 4
+      database.connectionPool.enabled: true
+      database.connectionPool.maxDbConnections: 8
+      database.connectionPool.maxClientConn: 250
+      gateway.envSecrets:
+        - litellm-license
+      gateway.volumes:
+        - name: redis-ca
+          secret:
+            secretName: redis-ca
+      gateway.volumeMounts:
+        - name: redis-ca
+          mountPath: /etc/litellm/redis-ca
+          readOnly: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_ADDRESS
+            value: unix:///var/run/litellm/collector.sock
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_BUFFER_SIZE
+            value: "250"
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_COLLECTOR_ON_UNAVAILABLE
+            value: drop
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: collector-socket
+            mountPath: /var/run/litellm
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: collector
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].image
+          value: ghcr.io/berriai/litellm-gateway:v1.102.0
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].command
+          value:
+            - python
+            - -m
+            - litellm.proxy.collector
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_JOB_ROLE
+            value: collector
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: CONFIG_FILE_PATH
+            value: /app/config/config.yaml
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+            value: "8"
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+            value: "250"
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: DATABASE_HOST
+            value: postgres.example.com
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_COLLECTOR_ADDRESS
+            value: unix:///var/run/litellm/collector.sock
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: NUM_WORKERS
+          any: true
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].envFrom
+          value:
+            - secretRef:
+                name: litellm-license
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: gateway-config
+            mountPath: /app/config/config.yaml
+            subPath: config.yaml
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: collector-socket
+            mountPath: /var/run/litellm
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: redis-ca
+            mountPath: /etc/litellm/redis-ca
+            readOnly: true
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].resources.limits.cpu
+          value: "1"
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: collector-socket
+            emptyDir:
+              sizeLimit: 1Mi
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.metrics[0]
+          value:
+            type: ContainerResource
+            containerResource:
+              name: cpu
+              container: gateway
+              target:
+                type: Utilization
+                averageUtilization: 70
+        template: gateway/hpa.yaml
+
+  - it: uses loopback tcp without a socket volume and keeps the pod-wide cpu metric when asked
+    set:
+      gateway.collector.enabled: true
+      gateway.collector.address: tcp://127.0.0.1:4010
+      gateway.collector.scaleOnGatewayContainerCpu: false
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_COLLECTOR_ADDRESS
+            value: tcp://127.0.0.1:4010
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: collector-socket
+          any: true
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.containers[1].volumeMounts
+          content:
+            name: collector-socket
+          any: true
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.metrics[0].type
+          value: Resource
+        template: gateway/hpa.yaml

helm/litellm/tests/connection_pool_tests.yaml

@@ -0,0 +1,204 @@
+suite: test in-container connection pool env vars
+templates:
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - backend/configmap.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: renders no pool env by default
+    template: gateway/deployment.yaml
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+          any: true
+
+  - it: enabled pool renders the three pgbouncer vars with the configured sizes
+    template: gateway/deployment.yaml
+    set:
+      gateway.numWorkers: 4
+      database.connectionPool.enabled: true
+      database.connectionPool.maxDbConnections: 8
+      database.connectionPool.maxClientConn: 250
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+            value: "8"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+            value: "250"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: NUM_WORKERS
+            value: "4"
+
+  - it: enabled pool uses the chart default sizes
+    template: gateway/deployment.yaml
+    set:
+      database.connectionPool.enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+            value: "20"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+            value: "1000"
+
+  - it: backend never gets the pool env
+    template: backend/deployment.yaml
+    set:
+      database.connectionPool.enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+          any: true
+
+  - it: collector sidecar gets the same pool env as the gateway container, the metrics sidecar none
+    template: gateway/deployment.yaml
+    set:
+      gateway.collector.enabled: true
+      gateway.metricsServer.enabled: true
+      database.connectionPool.enabled: true
+      database.connectionPool.maxDbConnections: 8
+      database.connectionPool.maxClientConn: 250
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: metrics
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+          any: true
+      - equal:
+          path: spec.template.spec.containers[2].name
+          value: collector
+      - contains:
+          path: spec.template.spec.containers[2].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[2].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+            value: "8"
+      - contains:
+          path: spec.template.spec.containers[2].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+            value: "250"
+
+  - it: collector sidecar gets no pool env when the pool is off
+    template: gateway/deployment.yaml
+    set:
+      gateway.collector.enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: collector
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_MAX_CLIENT_CONN
+          any: true
+
+  - it: pool with IAM auth renders both the pool and the token auth flag
+    template: gateway/deployment.yaml
+    set:
+      gateway.collector.enabled: true
+      database.connectionPool.enabled: true
+      database.writer.useIAMAuth: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: IAM_TOKEN_DB_AUTH
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: IAM_TOKEN_DB_AUTH
+            value: "true"
+
+  - it: pool with Entra auth renders both the pool and the token auth flag
+    template: gateway/deployment.yaml
+    set:
+      database.connectionPool.enabled: true
+      database.writer.useAzureEntraAuth: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: AZURE_POSTGRESQL_AUTH
+            value: "true"
+
+  - it: IAM auth without the pool still renders
+    template: gateway/deployment.yaml
+    set:
+      database.writer.useIAMAuth: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: IAM_TOKEN_DB_AUTH
+            value: "true"
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_PGBOUNCER_ENABLED
+          any: true

helm/litellm/tests/database_auth_tests.yaml

@@ -4,6 +4,7 @@ templates:
   - gateway/configmap.yaml
   - backend/deployment.yaml
   - backend/configmap.yaml
+  - migrations-job.yaml
 values:
   - ./values/required.yaml
 tests:
@@ -67,6 +68,82 @@ tests:
             value: "true"
           any: true
 
+  - it: emits no TLS env by default
+    template: gateway/deployment.yaml
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_SSLMODE
+          any: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_SSLROOTCERT
+          any: true
+
+  - it: writer sslMode and sslRootCert reach gateway and backend as DATABASE_SSLMODE and DATABASE_SSLROOTCERT
+    templates:
+      - gateway/deployment.yaml
+      - backend/deployment.yaml
+    set:
+      database.writer.useIAMAuth: true
+      database.writer.sslMode: verify-full
+      database.writer.sslRootCert: /etc/ssl/certs/ca-certificates.crt
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_SSLMODE
+            value: verify-full
+          any: true
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_SSLROOTCERT
+            value: /etc/ssl/certs/ca-certificates.crt
+          any: true
+
+  - it: writer sslMode and sslRootCert reach the collector sidecar and the migrations job, which dial Postgres themselves
+    set:
+      gateway.collector.enabled: true
+      database.connectionPool.enabled: true
+      database.writer.sslMode: verify-full
+      database.writer.sslRootCert: /etc/ssl/certs/ca-certificates.crt
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: collector
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: DATABASE_SSLMODE
+            value: verify-full
+          any: true
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[1].env
+          content:
+            name: DATABASE_SSLROOTCERT
+            value: /etc/ssl/certs/ca-certificates.crt
+          any: true
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_SSLMODE
+            value: verify-full
+          any: true
+        template: migrations-job.yaml
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: DATABASE_SSLROOTCERT
+            value: /etc/ssl/certs/ca-certificates.crt
+          any: true
+        template: migrations-job.yaml
+
   - it: writer rejects both token sources at once
     template: gateway/deployment.yaml
     set:

helm/litellm/tests/hpa_workload_metrics_tests.yaml

@@ -0,0 +1,213 @@
+suite: test gateway HPA per-pod requests-per-second and tokens-per-second targets
+templates:
+  - gateway/hpa.yaml
+  - gateway/servicemonitor.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: scales on CPU and memory only by default
+    template: gateway/hpa.yaml
+    asserts:
+      - equal:
+          path: spec.metrics
+          value:
+            - type: Resource
+              resource:
+                name: cpu
+                target:
+                  type: Utilization
+                  averageUtilization: 70
+            - type: Resource
+              resource:
+                name: memory
+                target:
+                  type: Utilization
+                  averageUtilization: 80
+
+  - it: adds a requests-per-second Pods metric next to the resource metrics
+    template: gateway/hpa.yaml
+    set:
+      gateway.hpa.targetRequestsPerSecond: 90
+    asserts:
+      - lengthEqual:
+          path: spec.metrics
+          count: 3
+      - contains:
+          path: spec.metrics
+          content:
+            type: Resource
+            resource:
+              name: cpu
+              target:
+                type: Utilization
+                averageUtilization: 70
+      - equal:
+          path: spec.metrics[2]
+          value:
+            type: Pods
+            pods:
+              metric:
+                name: litellm_requests_per_second
+              target:
+                type: AverageValue
+                averageValue: "90"
+      - notContains:
+          path: spec.metrics
+          content:
+            type: Pods
+            pods:
+              metric:
+                name: litellm_tokens_per_second
+          any: true
+
+  - it: adds a tokens-per-second Pods metric on its own
+    template: gateway/hpa.yaml
+    set:
+      gateway.hpa.targetTokensPerSecond: 6M
+    asserts:
+      - lengthEqual:
+          path: spec.metrics
+          count: 3
+      - equal:
+          path: spec.metrics[2]
+          value:
+            type: Pods
+            pods:
+              metric:
+                name: litellm_tokens_per_second
+              target:
+                type: AverageValue
+                averageValue: "6M"
+      - notContains:
+          path: spec.metrics
+          content:
+            type: Pods
+            pods:
+              metric:
+                name: litellm_requests_per_second
+          any: true
+
+  - it: renders requests and tokens targets together and keeps CPU and memory
+    template: gateway/hpa.yaml
+    set:
+      gateway.hpa.targetRequestsPerSecond: 90
+      gateway.hpa.targetTokensPerSecond: 6000000
+    asserts:
+      - lengthEqual:
+          path: spec.metrics
+          count: 4
+      - equal:
+          path: spec.metrics[0].resource.name
+          value: cpu
+      - equal:
+          path: spec.metrics[1].resource.name
+          value: memory
+      - equal:
+          path: spec.metrics[2].pods.metric.name
+          value: litellm_requests_per_second
+      - equal:
+          path: spec.metrics[3].pods.metric.name
+          value: litellm_tokens_per_second
+      - equal:
+          path: spec.metrics[3].pods.target.averageValue
+          value: "6000000"
+
+  - it: scales on workload metrics alone when the resource targets are cleared
+    template: gateway/hpa.yaml
+    set:
+      gateway.hpa.targetCPUUtilizationPercentage: null
+      gateway.hpa.targetMemoryUtilizationPercentage: null
+      gateway.hpa.targetRequestsPerSecond: 90
+      gateway.hpa.targetTokensPerSecond: 6000000
+    asserts:
+      - lengthEqual:
+          path: spec.metrics
+          count: 2
+      - notContains:
+          path: spec.metrics
+          content:
+            type: Resource
+          any: true
+      - equal:
+          path: spec.metrics[0].pods.metric.name
+          value: litellm_requests_per_second
+      - equal:
+          path: spec.metrics[1].pods.metric.name
+          value: litellm_tokens_per_second
+      - notMatchRegexRaw:
+          pattern: per_minute
+
+  - it: ignores the per-minute keys, which the chart never shipped
+    template: gateway/hpa.yaml
+    set:
+      gateway.hpa.targetRequestsPerMinute: 5400
+      gateway.hpa.targetTokensPerMinute: 360000000
+    asserts:
+      - lengthEqual:
+          path: spec.metrics
+          count: 2
+      - notContains:
+          path: spec.metrics
+          content:
+            type: Pods
+          any: true
+
+  - it: renders no ServiceMonitor by default
+    template: gateway/servicemonitor.yaml
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: refuses a ServiceMonitor without the metrics server, whose http port needs a bearer token
+    template: gateway/servicemonitor.yaml
+    set:
+      gateway.serviceMonitor.enabled: true
+    asserts:
+      - failedTemplate:
+          errorPattern: gateway.serviceMonitor.enabled requires gateway.metricsServer.enabled
+
+  - it: scrapes each gateway pod through the metrics port
+    template: gateway/servicemonitor.yaml
+    release:
+      name: rel
+      namespace: llm
+    set:
+      gateway.serviceMonitor.enabled: true
+      gateway.metricsServer.enabled: true
+      gateway.serviceMonitor.labels:
+        release: kube-prometheus-stack
+    asserts:
+      - isKind:
+          of: ServiceMonitor
+      - equal:
+          path: metadata.labels.release
+          value: kube-prometheus-stack
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: rel
+            app.kubernetes.io/component: gateway
+      - equal:
+          path: spec.namespaceSelector.matchNames
+          value:
+            - llm
+      - equal:
+          path: spec.endpoints
+          value:
+            - port: metrics
+              path: /metrics/
+              interval: 15s
+              scrapeTimeout: 10s
+              scheme: http
+
+  - it: honours a custom scrape interval
+    template: gateway/servicemonitor.yaml
+    set:
+      gateway.serviceMonitor.enabled: true
+      gateway.serviceMonitor.interval: 30s
+      gateway.metricsServer.enabled: true
+    asserts:
+      - equal:
+          path: spec.endpoints[0].interval
+          value: 30s

helm/litellm/tests/ingress_controller_tests.yaml

@@ -97,6 +97,16 @@ tests:
                 name: RELEASE-NAME-litellm-gateway
                 port:
                   number: 4000
+      - contains:
+          path: spec.rules[0].http.paths
+          content:
+            path: /debug/memory/summary
+            pathType: Exact
+            backend:
+              service:
+                name: RELEASE-NAME-litellm-gateway
+                port:
+                  number: 4000
       - equal:
           path: spec.rules[0].http.paths[-1]
           value:

helm/litellm/tests/ingress_extra_paths_tests.yaml

@@ -288,6 +288,17 @@ tests:
       - failedTemplate:
           errorMessage: "ingress.extraPaths[0]: path /test with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it"
 
+  - it: rejects an entry that would take over the exact /debug/memory/summary route
+    set:
+      ingress.enabled: true
+      ingress.extraPaths:
+        - path: /debug/memory/summary
+          pathType: Exact
+          service: backend
+    asserts:
+      - failedTemplate:
+          errorMessage: "ingress.extraPaths[0]: path /debug/memory/summary with pathType Exact is already routed by this chart, and a duplicate would take it over rather than add to it"
+
   - it: allows a built-in path under a different pathType, which is a distinct rule
     set:
       ingress.enabled: true

helm/litellm/values.yaml

@@ -208,6 +208,11 @@ database:
       name: litellm-writer-secret
       usernameKey: username
       passwordKey: password
+    # libpq sslmode / sslrootcert applied to the writer and reader URLs (Prisma and the
+    # in-container PgBouncer); e.g. verify-full with /etc/ssl/certs/ca-certificates.crt for AWS RDS.
+    # sslRootCert on its own implies sslMode verify-full
+    sslMode: ""
+    sslRootCert: ""
 
   # Optional read-replica routing. When `reader.host` is set, the proxy routes
   # reads (find_*, count, group_by, query_raw/_first) to this endpoint while
@@ -225,6 +230,26 @@ database:
       usernameKey: username
       passwordKey: password
 
+  # In-container connection pool (PgBouncer, transaction mode) shared by every
+  # gateway worker in the pod. Without it each of the `gateway.numWorkers`
+  # workers opens its own Prisma pool straight to Postgres, so a pod's
+  # footprint against the database's connection ceiling is
+  # numWorkers x connection_limit and grows with every replica. With it, the
+  # pod holds at most maxDbConnections upstream connections no matter how many
+  # workers run; the workers connect to the pool over loopback, with no extra
+  # network hop. The chart emits LITELLM_PGBOUNCER_ENABLED /
+  # LITELLM_PGBOUNCER_MAX_DB_CONNECTIONS / LITELLM_PGBOUNCER_MAX_CLIENT_CONN on
+  # the gateway container and its collector sidecar only: the backend runs a
+  # single worker and the migrations Job must keep a direct connection. With
+  # `database.writer.useIAMAuth` or `useAzureEntraAuth` the pool mints and
+  # renews the database token itself, so the workers never see it. Starting profile for
+  # `gateway.numWorkers: 4` is maxDbConnections: 20, so a database with a
+  # 5000-connection ceiling fits roughly 200 gateway replicas.
+  connectionPool:
+    enabled: false
+    maxDbConnections: 20
+    maxClientConn: 1000
+
 # Optional Redis. Leave host empty to disable.
 #
 # This is the proxy's coordination store: cross-pod tpm/rpm rate limits, spend
@@ -284,6 +309,52 @@ gateway:
         memory: 128Mi
       limits:
         memory: 512Mi
+  # Prometheus Operator ServiceMonitor for the gateway pods. Scrapes the
+  # `<gateway>-metrics` Service, so it requires metricsServer above (the http
+  # port serves /metrics/ behind virtual-key auth). Every pod is its own scrape
+  # target, so the samples carry the `pod` label the per-pod autoscaling
+  # queries below group by.
+  serviceMonitor:
+    enabled: false
+    labels: {}
+    interval: 15s
+    scrapeTimeout: 10s
+  # Opt-in `collector` sidecar (same image, `python -m litellm.proxy.collector`)
+  # that runs the post-response spend pipeline (cost calculation, spend logs,
+  # spend counters, budget reservation reconciliation) so the uvicorn workers
+  # only serialise a compact event over loopback and go back to serving
+  # requests. It shares the pod's env, proxy config, in-container pgbouncer and
+  # Redis spend buffer, so the per-pod DB connection budget is unchanged.
+  # Delivery is at-most-once inside the pod: events already handed over are
+  # lost if the sidecar dies before writing them; events the workers cannot
+  # hand over follow `onUnavailable`.
+  collector:
+    enabled: false
+    # unix:///<dir>/<file>.sock (the <dir> becomes a shared emptyDir) or
+    # tcp://127.0.0.1:<port>
+    address: unix:///var/run/litellm/collector.sock
+    # Events each uvicorn worker holds in memory while the sidecar is slow or
+    # restarting.
+    bufferSize: 1000
+    # fallback: run the pipeline in the worker when the sidecar is unreachable
+    # or the buffer is full (spend stays exact, that request costs gateway CPU
+    # again). drop: count and discard the event instead (spend under-reports).
+    onUnavailable: fallback
+    # How long the workers keep pushing buffered events on shutdown, and how
+    # long the sidecar keeps serving open connections after SIGTERM.
+    drainTimeoutSeconds: 10
+    # Sized independently of the gateway container; the pipeline is CPU bound.
+    resources:
+      requests:
+        cpu: 500m
+        memory: 1Gi
+      limits:
+        cpu: "1"
+        memory: 2Gi
+    # With hpa.targetCPUUtilizationPercentage set, scale on an autoscaling/v2
+    # ContainerResource metric of the `gateway` container only, so the
+    # sidecar's CPU never drives inference replicas. Needs Kubernetes 1.30+.
+    scaleOnGatewayContainerCpu: true
   image:
     repository: ghcr.io/berriai/litellm-gateway
     tag: ""              # defaults to .Chart.AppVersion
@@ -340,6 +411,25 @@ gateway:
     #     policies:
     #       - { type: Percent, value: 100, periodSeconds: 30 }
     behavior: {}
+    # Opt-in per-pod workload targets, rendered as autoscaling/v2 `Pods` metrics
+    # named `litellm_requests_per_second` and `litellm_tokens_per_second` with an
+    # AverageValue target. They coexist with the CPU/memory targets above: the
+    # HPA scales on whichever metric asks for the most replicas. Kubernetes has
+    # no idea what a token is, so a Prometheus Adapter must serve those two
+    # names on custom.metrics.k8s.io from the proxy's counters, grouped by the
+    # scrape target's `pod` label (enable serviceMonitor above):
+    #   litellm_requests_per_second:
+    #     sum(rate(litellm_proxy_total_requests_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+    #   litellm_tokens_per_second:
+    #     sum(rate(litellm_total_tokens_metric_total{<<.LabelMatchers>>}[1m])) by (<<.GroupBy>>)
+    # rate() over [1m] is already per second, so no `* 60`. How fast the HPA
+    # reacts is set by that window, the scrape interval and the HPA sync period
+    # (15s by default), not by the unit: keep serviceMonitor.interval at 15s or
+    # faster so a 1m window holds at least 4 samples. averageValue takes SI
+    # suffixes, so "6M" is six million tokens per second per pod. Tokens are
+    # counted when a response completes, so TPS trails long streams.
+    targetRequestsPerSecond: ""
+    targetTokensPerSecond: ""
   # PodDisruptionBudget for the gateway pods. Set exactly one of
   # `minAvailable` / `maxUnavailable` (minAvailable wins if both are set;
   # enabling without either falls back to `maxUnavailable: 1`). Disabled by

litellm-proxy-extras/litellm_proxy_extras/_logging.py

@@ -40,4 +40,4 @@ if not logger.handlers:
             logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
         )
     logger.addHandler(handler)
-    logger.setLevel(logging.INFO)
+    logger.setLevel(os.getenv("LITELLM_LOG", "INFO").upper())

litellm-proxy-extras/litellm_proxy_extras/migrations/20260823000000_add_spend_logs_api_key_starttime_index/migration.sql

@@ -0,0 +1,2 @@
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogs_api_key_startTime_idx" ON "LiteLLM_SpendLogs"("api_key", "startTime");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260831120000_add_litellm_call_id_spend_logs/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN IF NOT EXISTS "litellm_call_id" TEXT;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260831120001_spend_logs_litellm_call_id_index/migration.sql

@@ -0,0 +1,12 @@
+-- CreateIndex (CONCURRENTLY)
+--
+-- Disclaimer:
+-- - CREATE INDEX CONCURRENTLY cannot run inside a transaction. This migration must stay a
+--   single statement so Prisma Migrate on PostgreSQL can apply it outside a transaction.
+-- - Builds are slower and use more I/O than a blocking CREATE INDEX; if the build is
+--   interrupted, Postgres may leave an INVALID index that must be dropped and recreated.
+-- - Do not edit this file after it has been applied to any database: Prisma checksums
+--   migrations; add a new migration instead.
+-- - Requires PostgreSQL that supports CONCURRENTLY with IF NOT EXISTS (use a new migration
+--   without IF NOT EXISTS if you must support older versions).
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_SpendLogs_litellm_call_id_idx" ON "LiteLLM_SpendLogs"("litellm_call_id");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260910000000_add_autorouter_session_baseline_models/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "LiteLLM_AutoRouterSession" ADD COLUMN IF NOT EXISTS "baseline_models" JSONB NOT NULL DEFAULT '{}';

litellm-proxy-extras/litellm_proxy_extras/migrations/20260910000000_skills_on_object_permission/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "skills" TEXT[] DEFAULT ARRAY[]::TEXT[];

litellm-proxy-extras/litellm_proxy_extras/migrations/20260913000000_add_tpd_limit/migration.sql

@@ -0,0 +1,14 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_BudgetTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DeletedTeamTable" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "tpd_limit" BIGINT;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_add_daily_response_time/migration.sql

@@ -0,0 +1,23 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "total_response_time_ms" BIGINT NOT NULL DEFAULT 0;
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "timed_requests" BIGINT NOT NULL DEFAULT 0;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260915000000_scope_jwt_key_mapping_by_issuer/migration.sql

@@ -0,0 +1,18 @@
+-- DropIndex
+DROP INDEX IF EXISTS "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_is_act_idx";
+
+-- DropIndex
+DROP INDEX IF EXISTS "LiteLLM_JWTKeyMapping_jwt_claim_name_jwt_claim_value_key";
+
+-- AlterTable
+-- NOT NULL DEFAULT '' (not nullable): Postgres unique constraints treat every
+-- NULL as distinct, so a nullable column would let multiple unscoped mappings
+-- collide on the same claim without a constraint violation. The constant
+-- default is a fast, metadata-only backfill for existing rows, not a rewrite.
+ALTER TABLE "LiteLLM_JWTKeyMapping" ADD COLUMN IF NOT EXISTS "jwt_issuer" TEXT NOT NULL DEFAULT '';
+
+-- CreateIndex
+CREATE INDEX IF NOT EXISTS "LiteLLM_JWTKeyMapping_jwt_issuer_jwt_claim_name_jwt_claim_v_idx" ON "LiteLLM_JWTKeyMapping"("jwt_issuer", "jwt_claim_name", "jwt_claim_value", "is_active");
+
+-- CreateIndex
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_JWTKeyMapping_jwt_issuer_jwt_claim_name_jwt_claim_v_key" ON "LiteLLM_JWTKeyMapping"("jwt_issuer", "jwt_claim_name", "jwt_claim_value");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260916000000_add_key_total_spend/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "total_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260917055603_add_policy_attachment_priority/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "LiteLLM_PolicyAttachmentTable" ADD COLUMN IF NOT EXISTS "priority" INTEGER;

litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py

@@ -37,11 +37,13 @@ raised it above the deploy default keeps that larger budget for deploy unless
 the deploy override says otherwise.
 """
 
+import importlib.util
 import math
 import os
 import shutil
 import signal
 import subprocess
+import sys
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
@@ -64,6 +66,7 @@ DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT = 600.0
 DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT = 600.0
 
 BOOTSTRAP_ARG = "--version"
+PRISMA_CONSOLE_SCRIPT = "prisma"
 
 
 @dataclass(frozen=True)
@@ -184,6 +187,28 @@ def _kill_process_group(process: "subprocess.Popen[str]") -> None:
         return
 
 
+def prisma_cli_available() -> bool:
+    """Whether some way of running the Prisma CLI exists: the console script on PATH or the importable package."""
+    if shutil.which(PRISMA_CONSOLE_SCRIPT) is not None:
+        return True
+    return importlib.util.find_spec(PRISMA_CONSOLE_SCRIPT) is not None
+
+
+def resolve_prisma_argv(argv: Sequence[str]) -> tuple[str, ...]:
+    """Route a bare ``prisma`` command through ``python -m prisma`` when the console script is not on PATH.
+
+    The console script and ``python -m prisma`` are the same entry point, but
+    only the module form survives an interpreter whose ``bin`` directory is
+    missing from PATH, which is how the proxy gets started under launchers and
+    init systems. Any other executable name is left untouched.
+    """
+    if not argv or argv[0] != PRISMA_CONSOLE_SCRIPT:
+        return tuple(argv)
+    if shutil.which(PRISMA_CONSOLE_SCRIPT) is not None:
+        return tuple(argv)
+    return (sys.executable, "-m", PRISMA_CONSOLE_SCRIPT, *argv[1:])
+
+
 def run_prisma(
     argv: Sequence[str],
     *,
@@ -200,7 +225,7 @@ def run_prisma(
     text unless ``stdout``/``stderr`` say otherwise.
     """
     with subprocess.Popen(
-        argv,
+        resolve_prisma_argv(argv),
         env=env,
         stdout=stdout,
         stderr=stderr,

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -17,6 +17,7 @@ model LiteLLM_BudgetTable {
   max_parallel_requests Int?
   tpm_limit     BigInt?
   rpm_limit     BigInt?
+  tpd_limit     BigInt?
   model_max_budget Json?
   budget_duration String?
   budget_reset_at DateTime?
@@ -133,6 +134,7 @@ model LiteLLM_TeamTable {
     max_parallel_requests Int?
     tpm_limit     BigInt?
     rpm_limit     BigInt?
+    tpd_limit     BigInt?
     budget_duration String? 
     budget_reset_at DateTime?
     blocked Boolean @default(false)
@@ -203,6 +205,7 @@ model LiteLLM_DeletedTeamTable {
     max_parallel_requests   Int?
     tpm_limit               BigInt?
     rpm_limit               BigInt?
+    tpd_limit               BigInt?
     budget_duration         String?
     budget_reset_at         DateTime?
     blocked                 Boolean  @default(false)
@@ -282,6 +285,7 @@ model LiteLLM_ObjectPermissionTable {
   mcp_toolsets          String[]       @default([]) // Toolset IDs granted to this key/team/user
   search_tools          String[]       @default([]) // search_tool_name values this key/team/user may call
   mcp_tool_search_enabled Boolean?
+  skills                String[]       @default([]) // Claude Code plugin names granted to this key/team beyond the public (enabled) set
   teams                 LiteLLM_TeamTable[]
   projects              LiteLLM_ProjectTable[]
   verification_tokens   LiteLLM_VerificationToken[]
@@ -422,6 +426,7 @@ model LiteLLM_VerificationToken {
     key_alias   String?
     soft_budget_cooldown Boolean @default(false) // key-level state on if budget alerts need to be cooled down
     spend      Float    @default(0.0)
+    total_spend Float   @default(0.0)
     expires    DateTime?
     models     String[]
     aliases    Json  @default("{}")
@@ -437,6 +442,7 @@ model LiteLLM_VerificationToken {
     blocked Boolean?
     tpm_limit     BigInt?
     rpm_limit     BigInt?
+    tpd_limit     BigInt?
     max_budget Float?    
     budget_duration String? 
     budget_reset_at DateTime?
@@ -482,6 +488,10 @@ model LiteLLM_VerificationToken {
 
 model LiteLLM_JWTKeyMapping {
     id                String   @id @default(uuid())
+    jwt_issuer        String   @default("")  // Scopes the mapping to one configured issuer; "" matches any issuer.
+                                              // Not nullable: Postgres unique constraints treat every NULL as
+                                              // distinct, so a nullable column would let multiple unscoped
+                                              // mappings collide on the same claim without a constraint violation.
     jwt_claim_name    String   // e.g. "sub", "email"
     jwt_claim_value   String   // The claim value to match
     token             String   // Hashed virtual key (FK)
@@ -494,8 +504,8 @@ model LiteLLM_JWTKeyMapping {
 
     litellm_verification_token LiteLLM_VerificationToken @relation(fields: [token], references: [token], onDelete: Cascade)
 
-    @@unique([jwt_claim_name, jwt_claim_value])
-    @@index([jwt_claim_name, jwt_claim_value, is_active])
+    @@unique([jwt_issuer, jwt_claim_name, jwt_claim_value])
+    @@index([jwt_issuer, jwt_claim_name, jwt_claim_value, is_active])
 }
 
 // Deprecated keys during grace period - allows old key to work until revoke_at
@@ -519,6 +529,7 @@ model LiteLLM_DeletedVerificationToken {
     key_alias               String?
     soft_budget_cooldown    Boolean  @default(false)
     spend                   Float    @default(0.0)
+    total_spend             Float    @default(0.0)
     expires                 DateTime?
     models                  String[]
     aliases                 Json     @default("{}")
@@ -533,6 +544,7 @@ model LiteLLM_DeletedVerificationToken {
     blocked                 Boolean?
     tpm_limit               BigInt?
     rpm_limit               BigInt?
+    tpd_limit               BigInt?
     max_budget              Float?
     budget_duration         String?
     budget_reset_at         DateTime?
@@ -658,12 +670,15 @@ model LiteLLM_SpendLogs {
   mcp_namespaced_tool_name String? 
   agent_id            String?
   proxy_server_request Json?     @default("{}")
+  litellm_call_id     String?
   created_at          DateTime   @default(now()) @map("created_at")
   updated_at          DateTime   @default(now()) @updatedAt @map("updated_at")
   @@index([startTime])
   @@index([startTime, request_id])
   @@index([end_user])
   @@index([session_id])
+  @@index([litellm_call_id])
+  @@index([api_key, startTime])
 }
 
 model LiteLLM_BudgetWindowSpend {
@@ -789,6 +804,8 @@ model LiteLLM_DailyUserSpend {
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
   failed_requests     BigInt      @default(0)
+  total_response_time_ms BigInt   @default(0)
+  timed_requests      BigInt      @default(0)
   created_at          DateTime @default(now())
   updated_at          DateTime @updatedAt
 
@@ -825,6 +842,8 @@ model LiteLLM_DailyOrganizationSpend {
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
   failed_requests     BigInt      @default(0)
+  total_response_time_ms BigInt   @default(0)
+  timed_requests      BigInt      @default(0)
   created_at          DateTime @default(now())
   updated_at          DateTime @updatedAt
 
@@ -861,6 +880,8 @@ model LiteLLM_DailyEndUserSpend {
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
   failed_requests     BigInt      @default(0)
+  total_response_time_ms BigInt   @default(0)
+  timed_requests      BigInt      @default(0)
   created_at          DateTime @default(now())
   updated_at          DateTime @updatedAt
   @@unique([end_user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@ -896,6 +917,8 @@ model LiteLLM_DailyAgentSpend {
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
   failed_requests     BigInt      @default(0)
+  total_response_time_ms BigInt   @default(0)
+  timed_requests      BigInt      @default(0)
   created_at          DateTime @default(now())
   updated_at          DateTime @updatedAt
   @@unique([agent_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name, endpoint])
@@ -931,6 +954,8 @@ model LiteLLM_DailyTeamSpend {
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
   failed_requests     BigInt      @default(0)
+  total_response_time_ms BigInt   @default(0)
+  timed_requests      BigInt      @default(0)
   ptu_flat_cost       Float    @default(0.0)
   created_at          DateTime @default(now())
   updated_at          DateTime @updatedAt
@@ -969,6 +994,8 @@ model LiteLLM_DailyTagSpend {
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
   failed_requests     BigInt      @default(0)
+  total_response_time_ms BigInt   @default(0)
+  timed_requests      BigInt      @default(0)
   created_at          DateTime @default(now())
   updated_at          DateTime @updatedAt
 
@@ -1352,6 +1379,7 @@ model LiteLLM_PolicyAttachmentTable {
   keys             String[]  @default([])  // Key aliases or patterns
   models           String[]  @default([])  // Model names or patterns
   tags             String[]  @default([])  // Tag patterns (e.g., ["healthcare", "prod-*"])
+  priority         Int?                    // Explicit execution order
   created_at       DateTime  @default(now())
   created_by       String?
   updated_at       DateTime  @default(now()) @updatedAt
@@ -1513,6 +1541,7 @@ model LiteLLM_AutoRouterSession {
   classifier_cost       Float    @default(0)
   classifier_cost_recorded_turns Int @default(0)
   tier_turns            Json     @default("{}")
+  baseline_models       Json     @default("{}")
 
   @@id([api_key, session_id, router_name])
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.95"
+version = "0.4.99"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.95"
+version = "0.4.99"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/ADDING_A_PROVIDER.md

@@ -1,29 +0,0 @@
-# Adding a provider / route to litellm-rust
-
-Everything for a route lives in `crates/core/src/<route>/`; `crates/core/src/messages` is the reference. A host (the axum gateway, the Python bridge) only calls the route's entrypoint.
-
-1. **Entrypoint** — `mod.rs`: `pub async fn <route>(request) -> CoreResult<Response>`, the Rust equivalent of `litellm.<route>()`, plus a `<route>_stream` variant when the route streams. It is the only thing a host touches.
-2. **Transform contract** — `transformation.rs`: a `…ProviderConfig` trait (URL build + request/response transforms) with types in `types.rs`.
-3. **Provider config** — `crates/core/src/providers/<provider>/<route>/transformation.rs`: implement that trait as a `const <PROVIDER>_<ROUTE>_CONFIG`, mirroring the Python provider tree. Add parity unit tests.
-4. **Prepare + handler** — `prepare.rs` resolves provider/model, credentials, auth headers, and URL, then transforms the request; `handler.rs` performs the provider call through the shared client in `client.rs` and transforms the response.
-
-## Coding standards
-
-Before writing new logic, look for an existing base to extend. When a change is
-“the same behavior for one more provider/endpoint/integration”, the codebase
-almost always already has a shared abstraction for it (for example, provider
-`BaseConfig` transformation classes in `litellm/llms/base_llm/`, shared
-helpers in `litellm_core_utils/`, typed request/response models, or factory
-functions). Find it first with a search, then add the new variant by inheriting
-from or composing that base, overriding only what genuinely differs (model
-name, parameter mapping, or auth).
-
-Never copy an existing implementation and edit it in place, and never hand-roll
-a parallel version of logic a base already provides. If you catch yourself
-writing a second copy of a pattern that exists twice already, stop and extract a
-base instead: put the shared shape in one place and make both call sites thin
-variants of it. The test for a good abstraction is that adding the next provider
-is a few declarative lines, not a new file of duplicated flow. Only diverge from
-the base when behavior is genuinely different, and say so explicitly in the PR.
-
-**Calling:** hosts invoke the core entrypoint — the Python bridge and the `ai-gateway` route service both call `litellm_core::messages::messages`. Never add a provider handler to `ai-gateway`. Register new modules in `lib.rs` / `mod.rs`, then run the commands under "Checks" in [CLAUDE.md](CLAUDE.md).

litellm-rust/AGENTS.md

@@ -1,44 +0,0 @@
-# AGENTS.md
-
-litellm-rust has five crates. A crate is a layer or shared foundation, not a route. Routes (ocr, realtime, chat) and providers (mistral, openai) are modules inside the layers.
-
-## Crates
-
-| Crate | Role |
-|-------|------|
-| litellm-core | The LiteLLM SDK in Rust. One public entrypoint per top-level call (`messages::messages()`), owning types, transforms, provider resolution, auth, and the provider HTTP call. Call it, get a typed response. |
-| litellm-config | Config-loading boundary. Returns resolved core deployment data and optionally delegates loading to Python. |
-| litellm-ai-gateway | The axum server (behind the `server` feature) plus the WebSocket hosts. Translates HTTP/WS to core entrypoints; owns no provider logic and no handlers. |
-| litellm-python-interop | Domain-neutral PyO3 foundation for GIL handling and typed Python/Serde conversion. |
-| litellm-python-bridge | PyO3 cdylib exposing LiteLLM Rust APIs to the Python SDK. Owns API registration, domain wiring, and Python exception mapping. |
-
-Dependency direction is acyclic: `litellm-config` depends on `litellm-core`, the gateway depends on both, and `litellm-python-bridge` depends on the domain layers and `litellm-python-interop`. The interop foundation depends on no LiteLLM domain crate.
-
-## Where a route lives
-
-A top-level LiteLLM call is a module under `crates/core/src/<route>/`, shaped like `messages`:
-
-```
-core/src/messages/
-  mod.rs             # pub async fn messages(..) -> CoreResult<..>  (+ messages_stream for SSE)
-  types.rs           # request/response types, MessagesRequest
-  transformation.rs  # the provider template trait
-  prepare.rs         # provider resolution, auth headers, URL
-  handler.rs         # the provider call
-  client.rs          # the shared reqwest client
-```
-
-Handlers never live in `ai-gateway`. `ocr`, `audio_transcription`, and `realtime` are still hosted there from before this rule; they move to `core` as they are touched.
-
-Adding a crate: default to a module. A new crate requires a real trigger: separate artifact (binary/cdylib), proc-macro, shared foundation, or publishable standalone. A new provider or route is none of these.
-
-Adding a crate fails crates/core/tests/workspace_crate_allowlist.rs until you update its allowlist and this file — intentional.
-
-## Style
-
-All Rust in `litellm-rust/` follows the official Rust Style Guide:
-https://doc.rust-lang.org/style-guide/
-
-`rustfmt` implements its formatting by default, so run `cargo fmt` before committing; CI gates every PR on `cargo fmt --check`. Do not hand-format against rustfmt or add a `rustfmt.toml` that diverges from the default style.
-
-Beyond formatting, follow the guide's naming and idiom conventions rustfmt cannot auto-apply: `snake_case` items/functions/modules, `UpperCamelCase` types/traits/variants, `SCREAMING_SNAKE_CASE` constants/statics (acronyms as one word, e.g. `HttpClient`), and the import grouping and item ordering it prescribes. See CLAUDE.md for the detailed version.

litellm-rust/CLAUDE.md

@@ -1,189 +0,0 @@
-# CLAUDE.md
-
-This file defines the rules for Rust work in LiteLLM.
-
-## Provider Coding Standards
-
-Before writing new logic, look for an existing base to extend. When a change is
-“the same behavior for one more provider/endpoint/integration”, the codebase
-almost always already has a shared abstraction for it (for example, provider
-`BaseConfig` transformation classes in `litellm/llms/base_llm/`, shared
-helpers in `litellm_core_utils/`, typed request/response models, or factory
-functions). Find it first with a search, then add the new variant by inheriting
-from or composing that base, overriding only what genuinely differs (model
-name, parameter mapping, or auth).
-
-Never copy an existing implementation and edit it in place, and never hand-roll
-a parallel version of logic a base already provides. If you catch yourself
-writing a second copy of a pattern that exists twice already, stop and extract a
-base instead: put the shared shape in one place and make both call sites thin
-variants of it. The test for a good abstraction is that adding the next provider
-is a few declarative lines, not a new file of duplicated flow. Only diverge from
-the base when behavior is genuinely different, and say so explicitly in the PR.
-
-## Crates (see AGENTS.md)
-
-`litellm-core` **is** the LiteLLM SDK in Rust: it makes the LLM call.
-`litellm-config` is the config-loading boundary and returns resolved core types.
-`litellm-ai-gateway` is an HTTP/WebSocket server in front of it, and
-`litellm-python-bridge` exposes it to the Python SDK. `litellm-python-interop`
-holds domain-neutral PyO3 primitives shared by Python-facing Rust code. A crate
-is a layer or shared foundation, not a route; add modules, not crates.
-
-## Core Boundary
-
-`litellm-core` owns the whole call. The Rust equivalent of `litellm.messages()`
-is `litellm_core::messages::messages(request).await`: you call it, it does the
-provider call, and you get a typed non-streaming response back.
-
-Route-level Rust structure mirrors LiteLLM's Python responsibilities:
-- `core/src/<route>/` owns the route end to end: the public entrypoint fn named
-  after the route in `mod.rs`, the request/response types (`types.rs`), the
-  provider template trait (`transformation.rs`), the provider/auth/URL
-  resolution (`prepare.rs`), the HTTP client (`client.rs`), and the handler that
-  performs the call (`handler.rs`). `core/src/messages` is the reference.
-- `core/src/providers/<provider>/<route>/transformation.rs` owns the
-  provider-specific transform. For Anthropic Messages, this means
-  `core/src/providers/anthropic/messages/transformation.rs`.
-- Handlers live in `core`, never in a host. `ai-gateway` must not contain a
-  route handler that talks to a provider; its axum route reads the HTTP request,
-  picks a deployment, and calls the `core` entrypoint. `python-bridge` marshals
-  Python objects and calls the same entrypoint.
-
-Streaming keeps the same shape: the route entrypoint has a `<route>_stream`
-variant in `core` that returns the upstream response so a host can splice it to
-its own caller; the host still owns no provider logic.
-
-Call-hook and lifecycle instrumentation, including phase timing, usage
-accumulation, and callback payload construction, always lives in `core`.
-Hosts feed observed events into core and dispatch the completed payloads through
-their I/O logger; hosts must not own callback orchestration.
-
-Allowed in `core`:
-- The public entrypoint for a top-level LiteLLM call
-- Request/response transforms and stream chunk normalization
-- Provider resolution, auth header construction, and URL building
-- The provider HTTP call itself, through a shared reused client with connect and
-  request timeouts
-- Shared data types and validation errors
-- Deterministic token/cost helper logic
-
-Not allowed in `core`:
-- Serving HTTP: axum routes, extractors, and transport concerns stay in the host
-- Filesystem access
-- Database access
-- Config file reading and rollout state
-- Logging callbacks, spend writes, or custom callbacks
-- Global mutable runtime state
-
-Env reads in `core` are limited to credential fallback inside a route's
-`prepare.rs` (the `env_lookup` closure), mirroring what the Python SDK does when
-no key is passed. Everything else config-shaped is resolved by the host and
-passed in.
-
-Routes still hosted in `ai-gateway` (`ocr`, `audio_transcription`, `realtime`)
-predate this rule and are being moved into `core` route modules; do not add new
-ones there, and prefer moving one when you touch it.
-
-Python owns rollout state and fallback while Rust is being introduced. Rust
-paths must be off by default until parity tests prove equivalence with Python.
-A new provider/route may instead be implemented rust-only with no Python
-reference; then the Python interface is a thin dispatch that calls Rust with no
-fallback, and you state the rust-only choice explicitly in the PR. Either way
-the Python side stays minimal (it only marshals inputs and calls the Rust
-interface), never add a per-route feature flag, and never push provider
-dispatch into `litellm/main.py`; put it in a thin dispatch class under
-`litellm/llms/<provider>/<route>/`.
-
-## Production Bar
-
-Rust code in this workspace is held to a strict parity and robustness bar from
-the first PR:
-
-- Correctness parity is proven with tests. Do not rely on README claims or
-  manual inspection for a port that mirrors Python behavior.
-- Every provider transform must have unit tests for supported-parameter
-  filtering, request body shape, response normalization, missing/null fields,
-  and bad-input errors.
-- When Rust is exposed through Python, add Python tests that prove disabled,
-  enabled, and unavailable-bridge fallback behavior.
-- Avoid panics on user/provider input. Return typed errors and let the host map
-  them to Python exceptions or HTTP responses.
-- OCR handles documents that often contain personal data. Do not log document
-  contents, base64 payloads, provider response bodies, or secrets.
-- Error messages must be useful but data-minimized. Truncate or sanitize any
-  upstream body before it crosses a host boundary.
-- Treat empty or whitespace-only credentials, URLs, and config values as absent
-  at the host/config resolution layer.
-- Preserve Python output shape intentionally. If a field is always serialized as
-  `null` for Python parity, leave a short comment explaining that parity choice.
-
-## Network I/O Rules
-
-These rules apply to every module that executes network I/O, whether it is a
-`core` route handler or a host such as `ai-gateway`:
-
-- Set connect and full-request timeouts. No unbounded waits.
-- Reuse HTTP clients; do not construct clients per request.
-- Prefer rustls TLS for portable Python wheels and Linux images unless there is
-  a documented reason not to.
-- Add request IDs and structured tracing at the host layer, without logging OCR
-  document contents or secrets.
-- Do not echo raw upstream response bodies to callers. Sanitize and bound them.
-- Avoid `expect`/`unwrap` in server startup and request paths unless the panic is
-  impossible by construction and documented.
-
-## Rust Style Guide
-
-All Rust in `litellm-rust/` follows the official Rust Style Guide:
-https://doc.rust-lang.org/style-guide/
-
-`rustfmt` implements the guide's formatting rules by default, so the mechanical
-side is enforced for you: run `cargo fmt` before committing and CI gates every
-PR on `cargo fmt --check` (see Checks). Do not hand-format against rustfmt or add
-a `rustfmt.toml` that diverges from the default style; the default style *is* the
-guide.
-
-The guide also covers conventions rustfmt cannot auto-apply; follow these too:
-- Naming: `snake_case` for items, functions, and modules; `UpperCamelCase` for
-  types, traits, and enum variants; `SCREAMING_SNAKE_CASE` for constants and
-  statics; acronyms count as one word (`HttpClient`, not `HTTPClient`).
-- Ordering and grouping the guide prescribes: imports grouped std / external /
-  crate-local, derives before other attributes, and consistent item order.
-- Idioms the guide recommends over the formatter fighting you (e.g. prefer
-  restructuring an over-long expression rather than forcing an awkward wrap).
-
-## Constants
-
-Magic numbers and fixed strings go in a crate-level `constants.rs`, never
-hardcoded inline — the Rust mirror of Python's `litellm/constants.py`.
-
-- Each crate that needs them has `src/constants.rs` (declared `mod constants;`);
-  import from it (`use crate::constants::...`). Don't scatter `const` values at
-  the top of feature modules.
-- An env-overridable tunable still lives in `constants.rs` as its `DEFAULT_*`
-  value; the env read (with fallback to that default) happens at the host/config
-  resolution layer, not in `core`/`providers`.
-- Exception: a value that is purely local to one function and has no meaning
-  elsewhere may stay inline, but prefer `constants.rs` when in doubt.
-
-## Checks
-
-Run these before pushing Rust changes. The same checks run in GitHub Actions
-for changes under `litellm-rust/`.
-
-```bash
-cd litellm-rust
-cargo fmt --check
-cargo clippy --workspace --all-targets -- -D warnings
-cargo clippy -p litellm-core --all-targets --features bedrock-auth -- -D warnings
-# the ai-gateway binary + server code is behind the `server` feature
-cargo clippy -p litellm-ai-gateway --all-targets --all-features -- -D warnings
-cargo test --workspace
-cargo test -p litellm-core --features bedrock-auth
-# the `auth`, `routes`, `state` and `realtime` tests only exist under `server`
-cargo test -p litellm-ai-gateway --features server
-```
-
-When a Rust path is exposed through Python, add Python parity tests that compare
-the existing Python output with the Rust-backed output.