Merge remote-tracking branch 'origin/main' into fix/bedrock-guardrail-role-filter-fallback

# Conflicts:
#	litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
#	tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py

.circleci/requirements.txt

@@ -1,21 +0,0 @@
-# used by CI/CD testing
-openai==1.100.1
-python-dotenv
-tiktoken
-importlib_metadata
-cohere
-redis==5.2.1
-redisvl==0.4.1
-anthropic
-orjson==3.10.15 # fast /embedding responses
-pydantic==2.12.5
-google-cloud-aiplatform==1.133.0
-google-cloud-iam==2.19.1
-fastapi-sso==0.16.0 
-uvloop==0.21.0
-mcp==1.26.0    # for MCP server
-semantic_router==0.1.10 # for auto-routing with litellm
-fastuuid==0.14.0
-responses==0.25.7  # for proxy client tests
-pytest-retry==1.6.3  # for automatic test retries
-litellm-proxy-extras  # for prisma migrations

.devcontainer/post-create.sh

@@ -1,17 +1,17 @@
 #!/usr/bin/env bash
 set -e
 
-echo "[post-create] Installing poetry via pip"
-python -m pip install --upgrade pip
-python -m pip install poetry
+echo "[post-create] Installing uv"
+curl -LsSf https://astral.sh/uv/0.10.9/install.sh | env UV_NO_MODIFY_PATH=1 sh
+export PATH="$HOME/.local/bin:$PATH"
 
-echo "[post-create] Installing Python dependencies (poetry)"
-poetry install --with dev --extras proxy
+echo "[post-create] Installing Python dependencies (uv)"
+uv sync --frozen --group proxy-dev --extra proxy
 
 echo "[post-create] Generating Prisma client"
-poetry run prisma generate
+uv run --no-sync prisma generate
 
 echo "[post-create] Installing npm dependencies"
 cd ui/litellm-dashboard && npm ci
 
-echo "[post-create] Done"
+echo "[post-create] Done"

.gitguardian.yaml

@@ -37,7 +37,7 @@ secret:
     - "docs/**"
     - "**/*.md"
     - "**/*.lock"
-    - "poetry.lock"
+    - "uv.lock"
     - "package-lock.json"
 
   # Ignore security incidents with the SHA256 of the occurrence (false positives)

.github/actions/helm-oci-chart-releaser/action.yml

@@ -1,94 +0,0 @@
-name: Helm OCI Chart Releaser
-description: Push Helm charts to OCI-based (Docker) registries
-author: sergeyshaykhullin
-branding:
-  color: yellow
-  icon: upload-cloud
-inputs:
-  name:
-    required: true
-    description: Chart name
-  repository:
-    required: true
-    description: Chart repository name
-  tag:
-    required: true
-    description: Chart version
-  app_version:
-    required: true
-    description: App version
-  path:
-    required: false
-    description: Chart path (Default 'charts/{name}')
-  registry:
-    required: true
-    description: OCI registry
-  registry_username:
-    required: true
-    description: OCI registry username
-  registry_password:
-    required: true
-    description: OCI registry password
-  update_dependencies:
-    required: false
-    default: 'false'
-    description: Update chart dependencies before packaging (Default 'false')
-outputs:
-  image:
-    value: ${{ steps.output.outputs.image }}
-    description: Chart image (Default '{registry}/{repository}/{image}:{tag}')
-runs:
-  using: composite
-  steps:
-    - name: Helm | Setup
-      uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
-      with:
-        version: v3.20.0
-
-    - name: Helm | Login
-      shell: bash
-      env:
-        REGISTRY_PASSWORD: ${{ inputs.registry_password }}
-        REGISTRY_USERNAME: ${{ inputs.registry_username }}
-        REGISTRY: ${{ inputs.registry }}
-      run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY"
-
-    - name: Helm | Dependency
-      if: inputs.update_dependencies == 'true'
-      shell: bash
-      env:
-        CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
-      run: helm dependency update "$CHART_PATH"
-
-    - name: Helm | Package
-      shell: bash
-      env:
-        CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
-        TAG: ${{ inputs.tag }}
-        APP_VERSION: ${{ inputs.app_version }}
-      run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION"
-
-    - name: Helm | Push
-      shell: bash
-      env:
-        NAME: ${{ inputs.name }}
-        TAG: ${{ inputs.tag }}
-        REGISTRY: ${{ inputs.registry }}
-        REPOSITORY: ${{ inputs.repository }}
-      run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}"
-
-    - name: Helm | Logout
-      shell: bash
-      env:
-        REGISTRY: ${{ inputs.registry }}
-      run: helm registry logout "$REGISTRY"
-
-    - name: Helm | Output
-      id: output
-      shell: bash
-      env:
-        REGISTRY: ${{ inputs.registry }}
-        REPOSITORY: ${{ inputs.repository }}
-        NAME: ${{ inputs.name }}
-        TAG: ${{ inputs.tag }}
-      run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT

.github/pull_request_template.md

@@ -2,6 +2,10 @@
 
 <!-- e.g. "Fixes #000" -->
 
+## Linear ticket
+
+<!-- if you are an internal contributor, add the Linear ticket e.g. "Resolves LIT-1234" to magically link the Linear ticket to the GitHub PR -->
+
 ## Pre-Submission checklist
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
@@ -32,6 +36,13 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 - [ ] **Merge / cherry-pick CI run**  
        Links:
 
+## Screenshots / Proof of Fix
+
+<!-- Include screenshots, screen recordings, or log output demonstrating that your changes work as expected.
+     For bug fixes: show reproduction before the fix and passing behavior after.
+     For new features: show the feature working end-to-end.
+     For UI changes: include before/after screenshots. -->
+
 ## Type
 
 <!-- Select the type of Pull Request -->

.github/scripts/close_duplicate_issues.py

@@ -42,7 +42,9 @@ def gh(*args: str) -> str:
 def fetch_open_issues(repo: str | None) -> list[dict]:
     """Fetch all open issues (excluding PRs) via gh api --paginate."""
     if repo:
-        endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+        endpoint = (
+            f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+        )
     else:
         endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
     cmd = ["api", "--paginate", endpoint]
@@ -71,7 +73,9 @@ def close_as_duplicate(
     repo_args = ["--repo", repo] if repo else []
 
     if dry_run:
-        print(f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
+        print(
+            f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}"
+        )
         return
 
     # Add comment
@@ -115,7 +119,9 @@ def find_duplicate(
     return None
 
 
-def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
+def scan_all(
+    issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+) -> int:
     """Compare every issue against all older issues. Returns count of duplicates found."""
     # Sort oldest first
     issues.sort(key=lambda i: i["number"])
@@ -144,7 +150,11 @@ def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bo
 
 
 def check_single(
-    issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+    issue_number: int,
+    issues: list[dict],
+    threshold: float,
+    repo: str | None,
+    dry_run: bool,
 ) -> bool:
     """Check a single issue against all older open issues. Returns True if duplicate found."""
     target = None
@@ -178,13 +188,23 @@ def check_single(
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
+    parser = argparse.ArgumentParser(
+        description="Detect and close duplicate GitHub issues"
+    )
     mode = parser.add_mutually_exclusive_group(required=True)
     mode.add_argument("--scan", action="store_true", help="Scan all open issues")
     mode.add_argument("--issue-number", type=int, help="Check a single issue number")
-    parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
-    parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
-    parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
+    parser.add_argument(
+        "--threshold", type=float, default=0.85, help="Similarity threshold (0-1)"
+    )
+    parser.add_argument(
+        "--close",
+        action="store_true",
+        help="Actually close duplicates (default is dry-run)",
+    )
+    parser.add_argument(
+        "--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted."
+    )
     args = parser.parse_args()
 
     dry_run = not args.close
@@ -200,7 +220,9 @@ def main() -> None:
         count = scan_all(issues, args.threshold, args.repo, dry_run)
         print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
     else:
-        found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
+        found = check_single(
+            args.issue_number, issues, args.threshold, args.repo, dry_run
+        )
         sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
 
 

.github/scripts/scan_keywords.py

@@ -67,14 +67,13 @@ def send_webhook(webhook_url: str, payload: dict) -> None:
 def _excerpt(text: str, max_len: int = 400) -> str:
     if not text:
         return ""
-    
+
     # Keep original formatting
     if len(text) <= max_len:
         return text
     return text[: max_len - 1] + "…"
 
 
-
 def main() -> int:
     event = read_event_payload()
     if not event:
@@ -87,8 +86,19 @@ def main() -> int:
 
     # Keywords from env or defaults
     keywords_env = os.environ.get("KEYWORDS", "")
-    default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
-    keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
+    default_keywords = [
+        "azure",
+        "openai",
+        "bedrock",
+        "vertexai",
+        "vertex ai",
+        "anthropic",
+    ]
+    keywords = (
+        [k.strip() for k in keywords_env.split(",")]
+        if keywords_env
+        else default_keywords
+    )
 
     matches = detect_keywords(combined_text, keywords)
     found = bool(matches)
@@ -129,5 +139,3 @@ def main() -> int:
 
 if __name__ == "__main__":
     raise SystemExit(main())
-
-

.github/workflows/README.md

@@ -1,35 +0,0 @@
-# Simple PyPI Publishing
-
-A GitHub workflow to manually publish LiteLLM packages to PyPI with a specified version.
-
-## How to Use
-
-1. Go to the **Actions** tab in the GitHub repository
-2. Select **Simple PyPI Publish** from the workflow list
-3. Click **Run workflow**
-4. Enter the version to publish (e.g., `1.74.10`)
-
-## What the Workflow Does
-
-1. **Updates** the version in `pyproject.toml`
-2. **Copies** the model prices backup file
-3. **Builds** the Python package
-4. **Publishes** to PyPI
-
-## Prerequisites
-
-Make sure the following secret is configured in the repository:
-- `PYPI_PUBLISH_PASSWORD`: PyPI API token for authentication
-
-## Example Usage
-
-- Version: `1.74.11` → Publishes as v1.74.11
-- Version: `1.74.10-hotfix1` → Publishes as v1.74.10-hotfix1
-
-## Features
-
-- ✅ Manual trigger with version input
-- ✅ Automatic version updates in `pyproject.toml`
-- ✅ Repository safety check (only runs on official repo)
-- ✅ Clean package building and publishing
-- ✅ Success confirmation with PyPI package link 

.github/workflows/_test-unit-base.yml

@@ -51,37 +51,30 @@ jobs:
         with:
           python-version: "3.12"
 
-      - name: Install Poetry
-        run: pip install 'poetry==2.3.2'
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
 
-      - name: Cache Poetry dependencies
+      - name: Cache uv dependencies
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
-            ~/.cache/pypoetry
-            ~/.cache/pip
+            ~/.cache/uv
             .venv
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
           restore-keys: |
-            ${{ runner.os }}-poetry-
+            ${{ runner.os }}-uv-
 
       - name: Install dependencies
         run: |
-          poetry config virtualenvs.in-project true
-          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
-          poetry run pip install google-genai==1.22.0 \
-            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
-
-      - name: Setup litellm-enterprise
-        run: |
-          poetry run pip install --force-reinstall --no-deps -e enterprise/
+          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
       - name: Generate Prisma client
         env:
           PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
-          poetry run pip install nodejs-wheel-binaries==24.13.1
-          poetry run prisma generate --schema litellm/proxy/schema.prisma
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Run tests
         env:
@@ -90,7 +83,7 @@ jobs:
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
         run: |
-          poetry run pytest ${TEST_PATH:?} \
+          uv run --no-sync pytest ${TEST_PATH:?} \
             --tb=short -vv \
             --maxfail="${MAX_FAILURES}" \
             -n "${WORKERS}" \
@@ -98,7 +91,7 @@ jobs:
             --reruns-delay 1 \
             --dist=loadscope \
             --durations=20 \
-            --cov=litellm \
+            --cov=./litellm \
             --cov-report=xml:coverage.xml \
             --cov-config=pyproject.toml
 
@@ -139,4 +132,5 @@ jobs:
           use_oidc: true
           directory: coverage-reports
           root_dir: ${{ github.workspace }}
+          flags: ${{ inputs.artifact-name }}
           fail_ci_if_error: false

.github/workflows/_test-unit-services-base.yml

@@ -27,60 +27,44 @@ on:
         required: false
         type: number
         default: 10
-      enable-redis:
-        description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars"
-        required: false
-        type: boolean
-        default: false
       enable-postgres:
         description: "Start a local Postgres service container and run Prisma migrations"
         required: false
         type: boolean
         default: false
+      dist:
+        description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)"
+        required: false
+        type: string
+        default: "loadscope"
       artifact-name:
         description: "Unique name for the coverage artifact (must be unique per run)"
         required: false
         type: string
         default: "run"
-    secrets:
-      REDIS_HOST:
-        required: false
-      REDIS_PORT:
-        required: false
-      REDIS_PASSWORD:
-        required: false
-      DATABASE_URL:
-        required: false
-      POSTGRES_USER:
-        required: false
-      POSTGRES_PASSWORD:
-        required: false
 
 permissions:
   contents: read
 
+# The postgres service container below is spawned per-job on localhost and
+# destroyed with the job. Nothing outside the runner can reach it. The
+# user/password/database here are not secrets — they're bootstrap values
+# for a throwaway container — so we hardcode them instead of attaching
+# every matrix shard to a GHA environment just to read three "secrets"
+# (which also produces a "temporarily deployed to …" notification on the
+# PR timeline per shard per push).
 jobs:
   run:
     name: Run tests
     runs-on: ubuntu-latest
     timeout-minutes: ${{ inputs.timeout-minutes }}
-    # Environment is derived from the enable-* flags, not caller-controllable.
-    # This prevents callers from passing arbitrary environment names to bypass secret scoping.
-    # Note: Postgres service container always starts (GHA limitation), so any Redis job
-    # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis.
-    environment: >-
-      ${{
-        inputs.enable-redis && 'integration-redis-postgres' ||
-        inputs.enable-postgres && 'integration-postgres' ||
-        ''
-      }}
 
     services:
       postgres:
         image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14
         env:
-          POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+          POSTGRES_USER: litellm
+          POSTGRES_PASSWORD: litellm
           POSTGRES_DB: litellm_test
         ports:
           - 5432:5432
@@ -100,44 +84,37 @@ jobs:
         with:
           python-version: "3.12"
 
-      - name: Install Poetry
-        run: pip install 'poetry==2.3.2'
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
 
-      - name: Cache Poetry dependencies
+      - name: Cache uv dependencies
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
-            ~/.cache/pypoetry
-            ~/.cache/pip
+            ~/.cache/uv
             .venv
-          key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-uv-services-${{ hashFiles('uv.lock') }}
           restore-keys: |
-            ${{ runner.os }}-poetry-services-
+            ${{ runner.os }}-uv-services-
 
       - name: Install dependencies
         run: |
-          poetry config virtualenvs.in-project true
-          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
-          poetry run pip install google-genai==1.22.0 \
-            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
-
-      - name: Setup litellm-enterprise
-        run: |
-          poetry run pip install --force-reinstall --no-deps -e enterprise/
+          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
       - name: Generate Prisma client
         env:
           PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
-          poetry run pip install nodejs-wheel-binaries==24.13.1
-          poetry run prisma generate --schema litellm/proxy/schema.prisma
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Run Prisma migrations
         if: ${{ inputs.enable-postgres }}
         env:
-          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          DATABASE_URL: "postgresql://litellm:litellm@localhost:5432/litellm_test"
         run: |
-          poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+          uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
 
       - name: Run tests
         env:
@@ -145,31 +122,29 @@ jobs:
           MAX_FAILURES: ${{ inputs.max-failures }}
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
-          DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }}
-          REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }}
-          REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }}
-          REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }}
+          DIST: ${{ inputs.dist }}
+          DATABASE_URL: ${{ inputs.enable-postgres && 'postgresql://litellm:litellm@localhost:5432/litellm_test' || '' }}
         run: |
           if [ "${WORKERS}" = "0" ]; then
-            poetry run pytest ${TEST_PATH:?} \
+            uv run --no-sync pytest ${TEST_PATH:?} \
               --tb=short -vv \
               --maxfail="${MAX_FAILURES}" \
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
               --durations=20 \
-              --cov=litellm \
+              --cov=./litellm \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           else
-            poetry run pytest ${TEST_PATH:?} \
+            uv run --no-sync pytest ${TEST_PATH:?} \
               --tb=short -vv \
               --maxfail="${MAX_FAILURES}" \
               -n "${WORKERS}" \
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
-              --dist=loadscope \
+              --dist="${DIST}" \
               --durations=20 \
-              --cov=litellm \
+              --cov=./litellm \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           fi
@@ -211,4 +186,5 @@ jobs:
           use_oidc: true
           directory: coverage-reports
           root_dir: ${{ github.workspace }}
+          flags: ${{ inputs.artifact-name }}
           fail_ci_if_error: false

.github/workflows/auto_update_price_and_context_window.yml

@@ -17,12 +17,13 @@ jobs:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           persist-credentials: false
-      - name: Install Dependencies
-        run: |
-          pip install 'aiohttp==3.13.3'
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
       - name: Update JSON Data
         run: |
-          python ".github/workflows/auto_update_price_and_context_window_file.py"
+          uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py"
       - name: Create Pull Request
         run: |
           git add model_prices_and_context_window.json 

.github/workflows/check_duplicate_issues.yml

@@ -39,7 +39,7 @@ jobs:
         if: github.event.action == 'opened'
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Auto-close if high-confidence duplicate
         if: github.event.action == 'opened'

.github/workflows/codspeed.yml

@@ -34,13 +34,21 @@ jobs:
         with:
           python-version: "3.12"
 
-      - name: Install dependencies
-        run: |
-          pip install -e "."
-          pip install pytest pytest-codspeed==4.3.0
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
 
       - name: Run benchmarks
         uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
         with:
           mode: simulation
-          run: pytest tests/benchmarks/ --codspeed
+          run: >
+            env PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
+            uv run --frozen --no-default-groups
+            --with pytest==8.3.5
+            --with pytest-codspeed==4.3.0
+            pytest
+            -p pytest_codspeed.plugin
+            tests/benchmarks/
+            --codspeed

.github/workflows/create-release-branch.yml

@@ -0,0 +1,65 @@
+name: Create Release Branch
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted) — branch will be named release/<tag>"
+        required: true
+        type: string
+      commit_hash:
+        description: "Full 40-char commit SHA the branch should point to"
+        required: true
+        type: string
+  workflow_call:
+    inputs:
+      tag:
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Full 40-char commit SHA the branch should point to"
+        required: true
+        type: string
+
+permissions: {}
+
+jobs:
+  create-branch:
+    name: Create Release Branch
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Validate inputs
+        env:
+          TAG: ${{ inputs.tag }}
+          COMMIT_HASH: ${{ inputs.commit_hash }}
+        run: |
+          if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then
+            echo "::error::commit_hash must be a full 40-character commit SHA"
+            exit 1
+          fi
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
+            exit 1
+          fi
+
+      - name: Create release branch
+        env:
+          TAG: ${{ inputs.tag }}
+          COMMIT_HASH: ${{ inputs.commit_hash }}
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const tag = process.env.TAG;
+            const commitHash = process.env.COMMIT_HASH;
+            const branchName = `release/${tag}`;
+
+            await github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/heads/${branchName}`,
+              sha: commitHash,
+            });
+            core.info(`Created branch ${branchName} at ${commitHash}`);

.github/workflows/create-release.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. v1.83.0-stable)"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0-dev.2, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -30,8 +30,8 @@ jobs:
             echo "::error::commit_hash must be a full 40-character commit SHA"
             exit 1
           fi
-          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with vX.Y.Z"
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
             exit 1
           fi
 
@@ -45,6 +45,13 @@ jobs:
             const tag = process.env.TAG;
             const commitHash = process.env.COMMIT_HASH;
 
+            // Mark RC / dev / nightly / alpha / beta tags as GitHub pre-releases.
+            // Accept both PEP 440 (`.dev`) and SemVer (`-dev`) separators so tags
+            // like `1.84.0.dev2` and `1.84.0-dev.2` are both detected.
+            // PEP 440 post-releases (e.g. `1.84.0.post1`) and legacy `-stable[.patch.N]`
+            // are stable maintenance releases, not pre-releases.
+            const isPrerelease = /(?:rc|nightly|alpha|beta|[-.]dev)/i.test(tag);
+
             const cosignSection = [
               `## Verify Docker Image Signature`,
               ``,
@@ -89,7 +96,7 @@ jobs:
                 target_commitish: commitHash,
                 name: tag,
                 owner: context.repo.owner,
-                prerelease: false,
+                prerelease: isPrerelease,
                 repo: context.repo.repo,
                 tag_name: tag,
               });
@@ -102,6 +109,17 @@ jobs:
                 body: updatedBody,
                 draft: false,
               });
+
             } catch (error) {
               core.setFailed(error.message);
             }
+
+  create-branch:
+    name: Create Release Branch
+    needs: release
+    permissions:
+      contents: write
+    uses: ./.github/workflows/create-release-branch.yml
+    with:
+      tag: ${{ inputs.tag }}
+      commit_hash: ${{ inputs.commit_hash }}

.github/workflows/guard-fork-dependencies.yml

@@ -0,0 +1,140 @@
+name: Guard fork dependency changes
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
+    paths:
+      - "uv.lock"
+      - "pyproject.toml"
+      - "litellm-proxy-extras/pyproject.toml"
+      - "enterprise/pyproject.toml"
+
+permissions: {}
+
+jobs:
+  guard:
+    name: Block fork dependency changes
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    if: github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - name: Checkout base branch
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          ref: ${{ github.base_ref }}
+          persist-credentials: false
+          path: base
+
+      - name: Checkout PR head (read-only)
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+          path: pr
+
+      - name: Reject uv.lock changes
+        run: |
+          if ! diff -q base/uv.lock pr/uv.lock >/dev/null 2>&1; then
+            echo "::error::Fork PRs must not modify uv.lock. Dependency lockfile changes must come from a branch in the canonical repository."
+            exit 1
+          fi
+          echo "uv.lock is unchanged."
+
+      - name: Reject new dependencies in pyproject.toml files
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          # Write the checker script to a temp file to avoid shell quoting issues.
+          cat > /tmp/extract_deps.py << 'SCRIPT'
+          import re
+          import sys
+          import tomllib
+
+
+          def normalize(name: str) -> str:
+              return re.sub(r"[-_.]+", "-", name).lower()
+
+
+          def extract_dep_names(path: str) -> set[str]:
+              with open(path, "rb") as f:
+                  data = tomllib.load(f)
+
+              deps: set[str] = set()
+              pep508 = re.compile(r"^([A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?)")
+
+              # [project].dependencies
+              for spec in data.get("project", {}).get("dependencies", []):
+                  m = pep508.match(spec)
+                  if m:
+                      deps.add(normalize(m.group(1)))
+
+              # [project.optional-dependencies]
+              for group in data.get("project", {}).get("optional-dependencies", {}).values():
+                  for spec in group:
+                      m = pep508.match(spec)
+                      if m:
+                          deps.add(normalize(m.group(1)))
+
+              # [dependency-groups]
+              for group in data.get("dependency-groups", {}).values():
+                  for item in group:
+                      if isinstance(item, str):
+                          m = pep508.match(item)
+                          if m:
+                              deps.add(normalize(m.group(1)))
+
+              return deps
+
+
+          if __name__ == "__main__":
+              for name in sorted(extract_dep_names(sys.argv[1])):
+                  print(name)
+          SCRIPT
+
+          had_error=0
+
+          check_deps() {
+            local base_file="$1"
+            local pr_file="$2"
+            local label="$3"
+
+            if [ ! -f "$base_file" ] && [ ! -f "$pr_file" ]; then
+              return 0
+            fi
+
+            if [ ! -f "$base_file" ] && [ -f "$pr_file" ]; then
+              echo "::error::Fork PR introduces a new $label that does not exist on the base branch."
+              return 1
+            fi
+
+            base_deps=$(python3 /tmp/extract_deps.py "$base_file")
+            pr_deps=$(python3 /tmp/extract_deps.py "$pr_file")
+
+            new_deps=$(comm -13 <(echo "$base_deps") <(echo "$pr_deps"))
+
+            if [ -n "$new_deps" ]; then
+              echo "::error::Fork PR adds new dependencies in $label: $(echo $new_deps | tr '\n' ', ')"
+              echo "New packages detected:"
+              echo "$new_deps"
+              return 1
+            fi
+
+            echo "$label: no new dependencies."
+            return 0
+          }
+
+          check_deps base/pyproject.toml pr/pyproject.toml "pyproject.toml" || had_error=1
+          check_deps base/litellm-proxy-extras/pyproject.toml pr/litellm-proxy-extras/pyproject.toml "litellm-proxy-extras/pyproject.toml" || had_error=1
+          check_deps base/enterprise/pyproject.toml pr/enterprise/pyproject.toml "enterprise/pyproject.toml" || had_error=1
+
+          if [ "$had_error" -ne 0 ]; then
+            echo ""
+            echo "::error::Fork PRs must not add new dependencies. Please open an issue or coordinate with a maintainer to update dependencies from within the canonical repository."
+            exit 1
+          fi
+
+          echo "All pyproject.toml files passed dependency check."

.github/workflows/guard-main-branch.yml

@@ -0,0 +1,42 @@
+name: Guard main branch
+
+on:
+  pull_request:
+    branches:
+      - main
+  merge_group:
+
+permissions: {}
+
+# DO NOT RENAME the job's `name:` — it is referenced by GitHub branch
+# protection as a required status check on `main`. Renaming silently
+# breaks the gate.
+jobs:
+  guard:
+    name: Verify PR source branch
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    steps:
+      - name: Reject merge_group events
+        if: github.event_name == 'merge_group'
+        run: |
+          echo "::error::Merge queue is not supported for main. Disable merge queue or update this guard."
+          exit 1
+      - name: Check head branch name
+        env:
+          HEAD_REF: ${{ github.head_ref }}
+          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+          BASE_REPO: ${{ github.repository }}
+        run: |
+          echo "PR head repo: $HEAD_REPO"
+          echo "PR head branch: $HEAD_REF"
+          if [ "$HEAD_REPO" != "$BASE_REPO" ]; then
+            echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the 'litellm_oss_branch' branch instead."
+            exit 1
+          fi
+          if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then
+            echo "Allowed source branch."
+            exit 0
+          fi
+          echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_oss_branch' instead."
+          exit 1

.github/workflows/llm-translation-testing.yml

@@ -1,93 +0,0 @@
-name: LLM Translation Tests
-
-on:
-  workflow_dispatch:
-    inputs:
-      release_candidate_tag:
-        description: "Release candidate tag/version"
-        required: true
-        type: string
-  push:
-    tags:
-      - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1
-
-permissions:
-  contents: read
-
-jobs:
-  run-llm-translation-tests:
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-          ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.11"
-
-      - name: Install Poetry
-        run: |
-          pip install 'poetry==2.3.2'
-          poetry config virtualenvs.create true
-          poetry config virtualenvs.in-project true
-
-      - name: Restore Poetry dependencies cache
-        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
-        with:
-          path: |
-            ~/.cache/pypoetry
-            .venv
-          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-poetry-
-
-      - name: Install dependencies
-        run: |
-          poetry install --with dev
-          poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0'
-
-      - name: Create test results directory
-        run: mkdir -p test-results
-
-      - name: Run LLM Translation Tests
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
-          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
-          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
-          AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
-          RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }}
-          COMMIT_SHA: ${{ github.sha }}
-        run: |
-          python .github/workflows/run_llm_translation_tests.py \
-            --tag "$RC_TAG" \
-            --commit "$COMMIT_SHA" \
-            || true  # Continue even if tests fail
-
-      - name: Display test summary
-        if: always()
-        run: |
-          if [ -f "test-results/llm_translation_report.md" ]; then
-            echo "Test report generated successfully!"
-            echo "Artifact will contain:"
-            echo "- test-results/junit.xml (JUnit XML results)"
-            echo "- test-results/llm_translation_report.md (Beautiful markdown report)"
-          else
-            echo "Warning: Test report was not generated"
-          fi
-
-      - name: Upload test artifacts
-        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        if: always()
-        with:
-          name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}
-          path: test-results/
-          retention-days: 30

.github/workflows/mutation-test.yml

@@ -0,0 +1,131 @@
+name: "Mutation Test (manual)"
+
+# Manually-triggered mutation testing. Runs mutmut against the scope
+# configured in [tool.mutmut] in pyproject.toml (currently the
+# litellm/proxy/management_endpoints/ folder). Intended cadence is roughly
+# weekly — clicked from the Actions tab when someone wants a fresh report.
+#
+# Uploads a structured `mutation-report.md` (Meta ACH-style: original +
+# mutated function with `# MUTANT START`/`# MUTANT END` delimiters + the
+# existing tests + a task instruction) as a workflow artifact. Failures
+# do not block anything because nothing depends on this workflow.
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: mutation-test-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mutation:
+    name: Run mutmut
+    runs-on: ubuntu-latest
+    # Whole-folder mutation against ~15 files / ~7.5k LOC can take hours.
+    # 350 minutes is just under the GitHub-hosted job cap of 360 minutes.
+    timeout-minutes: 350
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
+      - name: Install dependencies
+        run: |
+          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      # mutmut 3.x runs tests inside a `mutants/` sandbox where it injects
+      # mutation trampolines. uv installs the project as editable by default,
+      # which puts the original source dir on sys.path via a .pth file and
+      # shadows the sandbox copy — so tests would never exercise the mutated
+      # code. Reinstalling non-editable removes the .pth shadow.
+      - name: Reinstall litellm non-editable (so mutants/ is not shadowed)
+        run: |
+          uv pip uninstall litellm
+          uv pip install . --no-deps
+
+      # pytest-retry's pytest_configure hook crashes with
+      # `INTERNALERROR: no option named 'filtered_exceptions'` when invoked
+      # via mutmut's in-process pytest.main() call. The entry-point name
+      # doesn't normalize cleanly with `-p no:<name>`, so just remove the
+      # package outright. Reruns are wrong for mutation testing anyway —
+      # rerunning a "failed" mutant test would mask which mutants are killed.
+      - name: Remove pytest plugins that conflict with mutmut
+        run: |
+          uv pip uninstall pytest-retry || true
+
+      - name: Run mutmut
+        env:
+          # Make the mutants/ sandbox win over site-packages on sys.path so the
+          # trampolined files are imported instead of the installed copy.
+          PYTHONPATH: ${{ github.workspace }}/mutants
+        run: |
+          set -o pipefail
+          mkdir -p mutants
+          uv run --no-sync --with mutmut==3.5.0 mutmut run 2>&1 | tee mutmut-run.log
+
+      # Generate the structured report. The script embeds the enclosing
+      # function source for each survivor (via Python AST) and includes the
+      # existing test files, so an LLM agent has enough context to write
+      # killing tests without further file lookups. Modeled on Meta's ACH
+      # prompt template (arXiv 2501.12862).
+      - name: Generate detailed mutation report
+        if: always()
+        run: |
+          set +e
+          uv run --no-sync --with mutmut==3.5.0 mutmut export-cicd-stats > /dev/null 2>&1
+          uv run --no-sync --with mutmut==3.5.0 mutmut results > mutmut-results.txt 2>&1
+          uv run --no-sync python scripts/mutation_report.py
+          # The full report can be very long for big test files; the run-page
+          # summary cuts off at 1 MB. Append the head of the report (summary
+          # + survivor list) and link out to the artifact for the full body.
+          {
+            head -c 900000 mutation-report.md
+            echo ""
+            echo ""
+            echo "_Full report (with embedded function bodies and test files) is in the workflow artifact._"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload mutmut artifacts
+        if: always()
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: mutmut-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            mutation-report.md
+            mutmut-results.txt
+            mutmut-run.log
+            mutants/mutmut-stats.json
+            mutants/mutmut-cicd-stats.json
+            mutants/litellm/proxy/management_endpoints/**/*.py
+          if-no-files-found: warn
+          retention-days: 14

.github/workflows/publish_to_pypi.yml

@@ -1,136 +0,0 @@
-name: Publish to PyPI
-
-on:
-  workflow_dispatch:
-
-jobs:
-  preflight-checks:
-    name: Preflight Checks
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-    # No environment — read-only checks, no approval needed
-    outputs:
-      needs_publish: ${{ steps.check-litellm.outputs.needs_publish }}
-      version: ${{ steps.check-litellm.outputs.version }}
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Check litellm version on PyPI
-        id: check-litellm
-        run: |
-          VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
-          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
-          echo "Checking if litellm $VERSION exists on PyPI..."
-
-          HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json")
-          if [ "$HTTP_STATUS" = "200" ]; then
-            echo "litellm $VERSION already exists on PyPI. Skipping publish."
-            echo "needs_publish=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "litellm $VERSION not found on PyPI. Publish needed."
-            echo "needs_publish=true" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Sanity check proxy-extras version
-        run: |
-          # Read pinned version from requirements.txt
-          REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt)
-          if [ -z "$REQ_VERSION" ]; then
-            echo "::error::Could not find litellm-proxy-extras version in requirements.txt"
-            exit 1
-          fi
-          echo "requirements.txt pins litellm-proxy-extras==$REQ_VERSION"
-
-          # Read pinned version from pyproject.toml dependency
-          PYPROJECT_VERSION=$(python3 -c "
-          import re
-          with open('pyproject.toml') as f:
-              content = f.read()
-          match = re.search(r'litellm-proxy-extras\s*=\s*\{version\s*=\s*\"([^\"]+)\"', content)
-          if match:
-              print(match.group(1).lstrip('^~>='))
-          else:
-              import sys
-              print('::error::Could not find litellm-proxy-extras dependency in pyproject.toml', file=sys.stderr)
-              sys.exit(1)
-          ")
-          echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION"
-
-          # Check that both pinned versions match
-          if [ "$REQ_VERSION" != "$PYPROJECT_VERSION" ]; then
-            echo "::error::Version mismatch: requirements.txt has $REQ_VERSION but pyproject.toml has $PYPROJECT_VERSION"
-            exit 1
-          fi
-
-          # Check that the pinned version exists on PyPI
-          echo "Checking if litellm-proxy-extras $REQ_VERSION exists on PyPI..."
-          HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$REQ_VERSION/json")
-          if [ "$HTTP_STATUS" != "200" ]; then
-            echo "::error::litellm-proxy-extras $REQ_VERSION is not published on PyPI yet. Publish it before releasing litellm."
-            exit 1
-          fi
-          echo "litellm-proxy-extras $REQ_VERSION exists on PyPI. Sanity check passed."
-
-  publish-litellm:
-    name: Publish litellm to PyPI
-    needs: preflight-checks
-    if: needs.preflight-checks.outputs.needs_publish == 'true'
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      id-token: write
-      contents: read
-    environment: pypi-publish
-
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Copy model prices backup
-        run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
-
-      - name: Install build tools
-        run: python -m pip install --upgrade pip build==1.4.2
-
-      - name: Build package
-        run: |
-          rm -rf build dist
-          python -m build
-
-      - name: Verify build artifacts
-        env:
-          EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }}
-        run: |
-          echo "Contents of dist/:"
-          ls -la dist/
-          # Ensure we have both sdist and wheel
-          ls dist/*.tar.gz
-          ls dist/*.whl
-          # Verify built version matches expected
-          ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || {
-            echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION"
-            ls dist/
-            exit 1
-          }
-
-      - name: Validate package metadata
-        run: |
-          pip install twine==6.2.0
-          twine check dist/*
-
-      - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0

.github/workflows/read_pyproject_version.yml

@@ -1,28 +0,0 @@
-name: Read Version from pyproject.toml
-
-on:
-  push:
-    branches:
-      - main # Change this to the default branch of your repository
-
-permissions:
-  contents: read
-
-jobs:
-  read-version:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Read version from pyproject.toml
-        id: read-version
-        run: |
-          version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
-          printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
-
-      - name: Display version
-        run: echo "Current version is $LITELLM_VERSION"

.github/workflows/results_stats.csv

@@ -1,27 +0,0 @@
-Date,"Ben 
-Ashley",Tom Brooks,Jimmy Cooney,"Sue 
-Daniels",Berlinda Fong,Terry Jones,Angelina Little,Linda Smith
-10/1,FALSE,TRUE,TRUE,TRUE,TRUE,TRUE,FALSE,TRUE
-10/2,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/3,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/4,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/5,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/6,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/7,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/8,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/9,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/10,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/11,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/12,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/13,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/14,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/15,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/16,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/17,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/18,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/19,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/20,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/21,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/22,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-10/23,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE,FALSE
-Total,0,1,1,1,1,1,0,1

.github/workflows/run_llm_translation_tests.py

@@ -325,7 +325,7 @@ def run_tests(test_path: str = "tests/llm_translation/",
     
     # Run pytest
     cmd = [
-        "poetry", "run", "pytest", test_path,
+        "uv", "run", "--no-sync", "pytest", test_path,
         f"--junitxml={junit_xml}",
         "-v",
         "--tb=short",
@@ -335,7 +335,7 @@ def run_tests(test_path: str = "tests/llm_translation/",
     
     # Add timeout if pytest-timeout is installed
     try:
-        subprocess.run(["poetry", "run", "python", "-c", "import pytest_timeout"], 
+        subprocess.run(["uv", "run", "--no-sync", "python", "-c", "import pytest_timeout"], 
                       capture_output=True, check=True)
         cmd.extend(["--timeout=300"])
     except:
@@ -436,4 +436,4 @@ if __name__ == "__main__":
         commit=args.commit
     )
     
-    sys.exit(exit_code)
+    sys.exit(exit_code)

.github/workflows/run_observatory_tests.yml

@@ -1,229 +0,0 @@
-name: Run Observatory Tests
-on:
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: "Docker image tag to test (e.g. v1.61.0.rc1)"
-        required: true
-        type: string
-      commit_hash:
-        description: "Commit hash (defaults to HEAD of current branch)"
-        required: false
-        type: string
-  workflow_call:
-    inputs:
-      tag:
-        description: "Docker image tag to test"
-        required: true
-        type: string
-      commit_hash:
-        description: "Commit hash of the release"
-        required: true
-        type: string
-
-permissions:
-  contents: read
-
-env:
-  LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
-
-jobs:
-  observatory-tests:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Validate tag input
-        env:
-          TAG: ${{ inputs.tag }}
-        run: |
-          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
-            echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
-            exit 1
-          fi
-
-      - name: Start LiteLLM container
-        env:
-          TAG: ${{ inputs.tag }}
-          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
-          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
-          WORKSPACE: ${{ github.workspace }}
-        run: |
-          docker run -d \
-            --name litellm-rc \
-            -p 4000:4000 \
-            -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
-            -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
-            -e AZURE_API_KEY="${AZURE_API_KEY}" \
-            -e AZURE_API_BASE="${AZURE_API_BASE}" \
-            "litellm/litellm:${TAG}" \
-            --config /app/config.yaml --port 4000
-
-      - name: Wait for LiteLLM health check
-        run: |
-          echo "Waiting for LiteLLM to be ready..."
-          for i in $(seq 1 30); do
-            if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
-              echo "LiteLLM is healthy"
-              exit 0
-            fi
-            echo "Attempt $i/30 - not ready yet, waiting 10s..."
-            sleep 10
-          done
-          echo "LiteLLM failed to start within 5 minutes"
-          docker logs litellm-rc
-          exit 1
-
-      - name: Start cloudflared tunnel
-        run: |
-          # Install cloudflared (pinned version + checksum)
-          curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
-          echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8  /usr/local/bin/cloudflared" | sha256sum -c -
-          chmod +x /usr/local/bin/cloudflared
-
-          # Start a quick tunnel (no account needed) and capture the URL
-          cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
-          CLOUDFLARED_PID=$!
-          echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
-
-          # Wait for tunnel URL to appear in logs
-          echo "Waiting for tunnel URL..."
-          for i in $(seq 1 30); do
-            TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
-            if [ -n "$TUNNEL_URL" ]; then
-              echo "Tunnel URL: $TUNNEL_URL"
-              echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
-              exit 0
-            fi
-            sleep 2
-          done
-          echo "Failed to get tunnel URL"
-          cat /tmp/cloudflared.log
-          exit 1
-
-      - name: Verify tunnel connectivity
-        run: |
-          echo "Testing tunnel at ${TUNNEL_URL}..."
-          # Quick tunnels need time for DNS propagation; retry to avoid
-          # transient NXDOMAIN (curl exit code 6) on first attempt.
-          for i in $(seq 1 10); do
-            if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then
-              echo "Tunnel is working (attempt $i)"
-              exit 0
-            fi
-            echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
-            sleep 5
-          done
-          echo "Tunnel failed to become reachable after 50s"
-          cat /tmp/cloudflared.log
-          exit 1
-
-      - name: Trigger observatory test run
-        id: trigger
-        env:
-          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
-          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
-        run: |
-          PAYLOAD=$(jq -n \
-            --arg url "${TUNNEL_URL}" \
-            --arg key "${LITELLM_MASTER_KEY}" \
-            '{
-              deployment_url: $url,
-              api_key: $key,
-              test_suite: "TestOAIAzureRelease",
-              models: ["gpt-4o-mini", "gpt-4o"]
-            }')
-          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
-            -H "Content-Type: application/json" \
-            -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
-            -d "$PAYLOAD")
-          HTTP_CODE=$(echo "$RESPONSE" | tail -1)
-          BODY=$(echo "$RESPONSE" | head -n -1)
-          echo "Response ($HTTP_CODE): $BODY"
-          if [ "$HTTP_CODE" -ge 400 ]; then
-            echo "Failed to trigger test run"
-            exit 1
-          fi
-
-          # Extract request_id for polling this specific run
-          REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
-          if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
-            echo "Failed to extract request_id from response"
-            exit 1
-          fi
-          echo "Request ID: $REQUEST_ID"
-          echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
-
-      - name: Poll for test completion
-        id: poll
-        env:
-          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
-          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
-          REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
-        run: |
-          TIMEOUT=900  # 15 minutes
-          INTERVAL=30
-          ELAPSED=0
-          while [ $ELAPSED -lt $TIMEOUT ]; do
-            STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
-              -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
-            RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
-            echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
-
-            if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
-              echo "Test finished with status: $RUN_STATUS"
-              echo "$STATUS" > /tmp/observatory_result.json
-              exit 0
-            fi
-
-            sleep $INTERVAL
-            ELAPSED=$((ELAPSED + INTERVAL))
-          done
-          echo "Timed out waiting for test to complete after ${TIMEOUT}s"
-          exit 1
-
-      - name: Verify test results
-        run: |
-          RESULT=$(cat /tmp/observatory_result.json)
-          echo "Full result: $RESULT"
-
-          STATUS=$(echo "$RESULT" | jq -r '.status')
-          TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
-          FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
-          ERROR=$(echo "$RESULT" | jq -r '.error // empty')
-
-          echo "Status: $STATUS"
-          echo "Test passed: $TEST_PASSED"
-          echo "Failure rate: $FAILURE_RATE"
-
-          if [ -n "$ERROR" ]; then
-            echo "Error: $ERROR"
-          fi
-
-          if [ "$STATUS" = "failed" ]; then
-            echo "Test run failed"
-            exit 1
-          fi
-
-          if [ "$TEST_PASSED" != "true" ]; then
-            echo "Tests did not pass (failure rate: $FAILURE_RATE)"
-            exit 1
-          fi
-
-          echo "All tests passed!"
-
-      - name: Print LiteLLM logs on failure
-        if: failure()
-        run: |
-          docker logs litellm-rc 2>/dev/null || true
-          cat /tmp/cloudflared.log 2>/dev/null || true
-
-      - name: Cleanup
-        if: always()
-        run: |
-          kill "$CLOUDFLARED_PID" 2>/dev/null || true
-          docker rm -f litellm-rc 2>/dev/null || true

.github/workflows/scan_duplicate_issues.yml

@@ -1,48 +0,0 @@
-name: Scan Duplicate Issues (One-Time)
-
-on:
-  workflow_dispatch:
-    inputs:
-      threshold:
-        description: "Similarity threshold (0-1)"
-        required: false
-        default: "0.85"
-      close:
-        description: "Actually close duplicates (false = dry run)"
-        required: false
-        type: boolean
-        default: false
-
-jobs:
-  scan:
-    runs-on: ubuntu-latest
-    permissions:
-      issues: write
-      contents: read
-    steps:
-      - name: Checkout scripts
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          sparse-checkout: .github/scripts
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.13"
-
-      - name: Scan for duplicate issues
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          INPUT_THRESHOLD: ${{ inputs.threshold }}
-          INPUT_CLOSE: ${{ inputs.close }}
-        run: |
-          CLOSE_FLAG=""
-          if [ "$INPUT_CLOSE" = "true" ]; then
-            CLOSE_FLAG="--close"
-          fi
-          python3 .github/scripts/close_duplicate_issues.py \
-            --scan \
-            --repo ${{ github.repository }} \
-            --threshold "$INPUT_THRESHOLD" \
-            $CLOSE_FLAG

.github/workflows/test-code-quality.yml

@@ -0,0 +1,128 @@
+name: Code Quality Checks
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  code-quality:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Checkout litellm-docs into docs/my-website (for documentation_tests)
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          repository: BerriAI/litellm-docs
+          path: docs/my-website
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
+      - name: Install dependencies
+        run: uv sync --frozen --all-groups --all-extras
+
+      - name: check_licenses
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_licenses.py
+
+      - name: check_provider_folders_documented
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
+
+      - name: router_code_coverage
+        run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
+
+      - name: test_chat_completion_imports
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_chat_completion_imports.py
+
+      - name: info_log_check
+        run: uv run --no-sync python ./tests/code_coverage_tests/info_log_check.py
+
+      - name: check_guardrail_apply_decorator
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_guardrail_apply_decorator.py
+
+      - name: test_ban_set_verbose
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_ban_set_verbose.py
+
+      - name: code_qa_check_tests
+        run: uv run --no-sync python ./tests/code_coverage_tests/code_qa_check_tests.py
+
+      - name: check_get_model_cost_key_performance
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_get_model_cost_key_performance.py
+
+      - name: test_proxy_types_import
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_proxy_types_import.py
+
+      - name: callback_manager_test
+        run: uv run --no-sync python ./tests/code_coverage_tests/callback_manager_test.py
+
+      - name: recursive_detector
+        run: uv run --no-sync python ./tests/code_coverage_tests/recursive_detector.py
+
+      - name: test_router_strategy_async
+        run: uv run --no-sync python ./tests/code_coverage_tests/test_router_strategy_async.py
+
+      - name: litellm_logging_code_coverage
+        run: uv run --no-sync python ./tests/code_coverage_tests/litellm_logging_code_coverage.py
+
+      - name: ensure_async_clients_test
+        run: uv run --no-sync python ./tests/code_coverage_tests/ensure_async_clients_test.py
+
+      - name: enforce_llms_folder_style
+        run: uv run --no-sync python ./tests/code_coverage_tests/enforce_llms_folder_style.py
+
+      - name: prevent_key_leaks_in_exceptions
+        run: uv run --no-sync python ./tests/code_coverage_tests/prevent_key_leaks_in_exceptions.py
+
+      - name: check_unsafe_enterprise_import
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
+
+      - name: ban_copy_deepcopy_kwargs
+        run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
+
+      - name: check_fastuuid_usage
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_fastuuid_usage.py
+
+      - name: memory_test
+        run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
+
+      - name: documentation_test_env_keys
+        run: uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
+
+      - name: documentation_test_router_settings
+        run: uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
+
+      - name: documentation_test_api_docs
+        run: uv run --no-sync python ./tests/documentation_tests/test_api_docs.py

.github/workflows/test-linting.yml

@@ -2,7 +2,11 @@ name: LiteLLM Linting
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read
@@ -24,26 +28,28 @@ jobs:
         with:
           python-version: "3.12"
 
-      - name: Install Poetry
-        run: pip install 'poetry==2.3.2'
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
 
       - name: Clean Python cache
         run: |
           find . -type d -name "__pycache__" -exec rm -rf {} + || true
           find . -name "*.pyc" -delete || true
 
-      - name: Check poetry.lock is up to date
+      - name: Check uv.lock is up to date
         run: |
-          poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1)
+          uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
 
       - name: Install dependencies
         run: |
-          poetry install --with dev
+          uv sync --frozen
 
       - name: Check Black formatting
         run: |
           cd litellm
-          poetry run black --check --exclude '/enterprise/' .
+          uv run --no-sync black --check --exclude '/enterprise/' .
           cd ..
 
       - name: Debug - Check file state
@@ -58,28 +64,28 @@ jobs:
       - name: Run Ruff linting
         run: |
           cd litellm
-          poetry run ruff check .
+          uv run --no-sync ruff check .
           cd ..
 
       - name: Print OpenAI version
         run: |
-          poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
+          uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
 
       - name: Run MyPy type checking
         run: |
           cd litellm
-          poetry run mypy . 
+          uv run --no-sync mypy .
           cd ..
 
       - name: Check for circular imports
         run: |
           cd litellm
-          poetry run python ../tests/documentation_tests/test_circular_imports.py
+          uv run --no-sync python ../tests/documentation_tests/test_circular_imports.py
           cd ..
 
       - name: Check import safety
         run: |
-          poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+          uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
 
   secret-scan:
     runs-on: ubuntu-latest
@@ -98,18 +104,21 @@ jobs:
         with:
           python-version: "3.12"
 
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
       - name: Run secret scan test
         run: |
-          pip install 'pytest==9.0.2'
-          pytest tests/litellm/test_no_hardcoded_secrets.py -v
+          uv run --frozen --with 'pytest==9.0.2' pytest tests/litellm/test_no_hardcoded_secrets.py -v
 
       - name: Run ggshield secret scan
         env:
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
         run: |
           if [ -n "$GITGUARDIAN_API_KEY" ]; then
-            pip install 'ggshield==1.48.0'
-            ggshield secret scan repo .
+            uv tool run --from 'ggshield==1.48.0' ggshield secret scan repo .
           else
             echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
           fi

.github/workflows/test-litellm-ui-build.yml

@@ -4,7 +4,11 @@ permissions:
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 jobs:
   build-ui:

.github/workflows/test-litellm.yml

@@ -1,53 +0,0 @@
-name: LiteLLM Mock Tests (folder - tests/test_litellm)
-
-# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs
-# the same tests in parallel across 10 jobs for faster CI times.
-# Kept for manual debugging only.
-on:
-  workflow_dispatch: # Manual trigger only
-  # pull_request:
-  #   branches: [ main ]
-
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 25
-
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Thank You Message
-        run: |
-          echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
-          echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Install Poetry
-        run: pip install 'poetry==2.3.2'
-
-      - name: Install dependencies
-        run: |
-          poetry lock
-          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
-          poetry run pip install "pytest-retry==1.6.3"
-          poetry run pip install 'pytest-xdist==3.8.0'
-          poetry run pip install "google-genai==1.22.0"
-          poetry run pip install "google-cloud-aiplatform==1.115.0"
-          poetry run pip install "fastapi-offline==1.7.3"
-          poetry run pip install "python-multipart==0.0.22"
-          poetry run pip install "openapi-core==0.23.0"
-      - name: Setup litellm-enterprise as local package
-        run: |
-          poetry run pip install --force-reinstall --no-deps -e enterprise/
-      - name: Run tests
-        run: |
-          poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

.github/workflows/test-mcp.yml

@@ -2,7 +2,11 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests)
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read
@@ -27,26 +31,16 @@ jobs:
         with:
           python-version: "3.12"
 
-      - name: Install Poetry
-        run: pip install 'poetry==2.3.2'
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
 
       - name: Install dependencies
         run: |
-          poetry lock
-          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
-          poetry run pip install "pytest==7.3.1"
-          poetry run pip install "pytest-retry==1.6.3"
-          poetry run pip install "pytest-cov==5.0.0"
-          poetry run pip install "pytest-asyncio==0.21.1"
-          poetry run pip install "respx==0.22.0"
-          poetry run pip install "pydantic==2.11.0"
-          poetry run pip install "mcp==1.25.0"
-          poetry run pip install 'pytest-xdist==3.8.0'
-
-      - name: Setup litellm-enterprise as local package
-        run: |
-          poetry run pip install --force-reinstall --no-deps -e enterprise/
+          uv lock --check
+          uv sync --frozen --group proxy-dev --extra proxy --extra semantic-router
 
       - name: Run MCP tests
         run: |
-          poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5
+          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5

.github/workflows/test-model-map.yaml

@@ -2,7 +2,11 @@ name: Validate model_prices_and_context_window.json
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-semgrep.yml

@@ -0,0 +1,39 @@
+name: Semgrep
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  semgrep:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Run Semgrep (custom rules)
+        run: uv tool run --from 'semgrep==1.157.0' semgrep scan --config .semgrep/rules . --error

.github/workflows/test-unit-caching-redis.yml

@@ -1,41 +0,0 @@
-name: "Unit Tests: Caching (Redis)"
-
-# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
-# This prevents external PRs from accessing Redis credentials.
-on:
-  push:
-    branches: [main, "litellm_*"]
-
-permissions:
-  contents: read
-  id-token: write
-  pull-requests: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  caching-redis:
-    uses: ./.github/workflows/_test-unit-services-base.yml
-    with:
-      # Redis-only tests that do NOT require provider API keys.
-      # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
-      # test_router_caching.py) are in Phase 3 integration workflows.
-      test-path: >-
-        tests/local_testing/test_dual_cache.py
-        tests/local_testing/test_redis_batch_optimizations.py
-        tests/local_testing/test_router_utils.py
-      workers: 2
-      reruns: 2
-      timeout-minutes: 20
-      enable-redis: true
-      enable-postgres: false
-      artifact-name: caching-redis
-    secrets:
-      REDIS_HOST: ${{ secrets.REDIS_HOST }}
-      REDIS_PORT: ${{ secrets.REDIS_PORT }}
-      REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
-      DATABASE_URL: ${{ secrets.DATABASE_URL }}
-      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test-unit-core-utils.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Core Utilities"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-documentation.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Documentation Validation"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read
@@ -21,47 +25,47 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Checkout litellm-docs into docs/my-website (for documentation_tests)
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          repository: BerriAI/litellm-docs
+          path: docs/my-website
+          persist-credentials: false
+
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
-      - name: Install Poetry
-        run: pip install 'poetry==2.3.2'
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
 
-      - name: Cache Poetry dependencies
+      - name: Cache uv dependencies
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
-            ~/.cache/pypoetry
-            ~/.cache/pip
+            ~/.cache/uv
             .venv
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
           restore-keys: |
-            ${{ runner.os }}-poetry-
+            ${{ runner.os }}-uv-
 
       - name: Install dependencies
         run: |
-          poetry config virtualenvs.in-project true
-          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
-          poetry run pip install google-genai==1.22.0 \
-            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
-
-      - name: Setup litellm-enterprise
-        run: |
-          poetry run pip install --force-reinstall --no-deps -e enterprise/
+          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
       - name: Generate Prisma client
         env:
           PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
-          poetry run pip install nodejs-wheel-binaries==24.13.1
-          poetry run prisma generate --schema litellm/proxy/schema.prisma
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       # Run the same documentation tests that CircleCI ran (as direct Python scripts)
       - name: Run documentation validation tests
         run: |
-          poetry run python ./tests/documentation_tests/test_env_keys.py
-          poetry run python ./tests/documentation_tests/test_router_settings.py
-          poetry run python ./tests/documentation_tests/test_api_docs.py
-          poetry run python ./tests/documentation_tests/test_circular_imports.py
+          uv run --no-sync python ./tests/documentation_tests/test_env_keys.py
+          uv run --no-sync python ./tests/documentation_tests/test_router_settings.py
+          uv run --no-sync python ./tests/documentation_tests/test_api_docs.py
+          uv run --no-sync python ./tests/documentation_tests/test_circular_imports.py

.github/workflows/test-unit-enterprise-routing.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Enterprise, Google GenAI & Routing"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-integrations.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Integrations (Callbacks & Logging)"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-llm-providers.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: LLM Provider Transformations"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-misc.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: MCP, Secrets, Containers & Misc"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-auth.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Auth & Key Management"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-db.yml

@@ -3,7 +3,7 @@ name: "Unit Tests: Proxy DB Operations"
 # Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
 on:
   push:
-    branches: [main, "litellm_*"]
+    branches: [main, "litellm_**"]
 
 permissions:
   contents: read
@@ -12,8 +12,74 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
+# Semantic matrix: each shard groups tests by concern (auth, server, logging, …)
+# rather than alphabetical letter ranges. Adding a new test file means adding it
+# to whichever group it belongs to, not reshuffling slices.
+#
+# Design targets:
+#   * Every shard runs in <= 7 minutes of wall-clock on the default runner.
+#     Most of a shard's time is pytest plugin load + xdist worker imports +
+#     pytest-cov instrumentation, not the tests themselves. Keeping per-shard
+#     work low and matching worker count to runner cores is what controls it.
+#   * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
+#     oversubscribes 2x and workers fight for CPU during their cold-start
+#     imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).
+#   * test_key_generate_prisma.py stays serial (workers=0) — it has event-loop
+#     conflicts with the logging worker when run in parallel.
+#   * test_proxy_utils.py runs as a single shard with --dist=worksteal so
+#     xdist balances its 188 parametrized cases across workers instead of
+#     pinning the whole file to one worker (the default --dist=loadscope
+#     behavior for single-file targets).
+#   * test_db_schema_migration.py is isolated because one test in it
+#     (test_aaaasschema_migration_check) takes ~170s — by itself it
+#     determines the shard's wall-clock floor.
 jobs:
+  # Fast guard — fails the workflow if a test_*.py file under
+  # tests/proxy_unit_tests/ is not referenced by any matrix entry below.
+  # The semantic-shard design (no catch-all "remaining" bucket) relies on
+  # every test file being explicitly assigned; this guard prevents a new
+  # file from silently dropping out of CI.
+  assert-shard-coverage:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+      - name: Assert every test_*.py is in a matrix shard
+        run: |
+          python3 - <<'PY'
+          import pathlib, sys, yaml
+          wf = yaml.safe_load(open(".github/workflows/test-unit-proxy-db.yml"))
+          matrix = wf["jobs"]["proxy-db"]["strategy"]["matrix"]["include"]
+          referenced = set()
+          for entry in matrix:
+              for token in entry["test-path"].split():
+                  if token.startswith("tests/proxy_unit_tests/"):
+                      referenced.add(pathlib.PurePosixPath(token).name)
+          actual = {p.name for p in pathlib.Path("tests/proxy_unit_tests").iterdir()
+                    if p.name.startswith("test_") and (p.suffix == ".py" or p.is_dir())
+                    and p.name != "test_configs"}
+          orphans = sorted(actual - referenced)
+          if orphans:
+              print("ERROR: the following files/dirs under tests/proxy_unit_tests/")
+              print("       are not assigned to any shard in test-unit-proxy-db.yml:")
+              for o in orphans:
+                  print(f"  - {o}")
+              print()
+              print("Add each to whichever semantic shard it belongs to.")
+              sys.exit(1)
+          print(f"OK: all {len(actual)} files assigned to a shard.")
+          PY
+
   proxy-db:
+    needs: assert-shard-coverage
+    # Display only the semantic shard name in the checks UI instead of GHA's
+    # default "proxy-db (key-generation, tests/proxy_unit_tests/…, 0, loadscope, 20)"
+    # which includes every matrix field and gets truncated past the test-path.
+    name: ${{ matrix.test-group }}
     permissions:
       contents: read
       id-token: write
@@ -22,29 +88,155 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Key generation tests must NOT run in parallel (event loop conflicts with logging worker)
+          # Must run serially — event-loop conflict with the logging worker.
           - test-group: key-generation
             test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py"
             workers: 0
-            timeout: 30
+            dist: loadscope
+            timeout: 20
+
+          # ---- auth: split into 2 shards ----
           - test-group: auth-checks
-            test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py"
-            workers: 8
-            timeout: 20
-          - test-group: remaining
-            test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py"
-            workers: 8
-            timeout: 20
+            test-path: >-
+              tests/proxy_unit_tests/test_auth_checks.py
+              tests/proxy_unit_tests/test_user_api_key_auth.py
+              tests/proxy_unit_tests/test_deprecated_key_grace_period.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: jwt-and-keys
+            test-path: >-
+              tests/proxy_unit_tests/test_jwt.py
+              tests/proxy_unit_tests/test_jwt_key_mapping.py
+              tests/proxy_unit_tests/test_proxy_custom_auth.py
+              tests/proxy_unit_tests/test_key_generate_dynamodb.py
+              tests/proxy_unit_tests/test_deployed_proxy_keygen.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- test_proxy_utils.py, single shard, worksteal distribution ----
+          - test-group: proxy-utils
+            test-path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            workers: 4
+            dist: worksteal
+            timeout: 15
+
+          # ---- proxy server: split into 2 shards ----
+          - test-group: proxy-server-core
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_server.py
+              tests/proxy_unit_tests/test_proxy_server_keys.py
+              tests/proxy_unit_tests/test_proxy_server_caching.py
+              tests/proxy_unit_tests/test_proxy_server_langfuse.py
+              tests/proxy_unit_tests/test_proxy_server_spend.py
+              tests/proxy_unit_tests/test_aproxy_startup.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: proxy-runtime
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_config_unit_test.py
+              tests/proxy_unit_tests/test_proxy_routes.py
+              tests/proxy_unit_tests/test_proxy_gunicorn.py
+              tests/proxy_unit_tests/test_server_root_path.py
+              tests/proxy_unit_tests/test_proxy_pass_user_config.py
+              tests/proxy_unit_tests/test_proxy_token_counter.py
+              tests/proxy_unit_tests/test_request_size_limit_middleware.py
+              tests/proxy_unit_tests/test_multipart_bypass_repro.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- logging: split into 2 shards ----
+          - test-group: custom-logging
+            test-path: >-
+              tests/proxy_unit_tests/test_custom_callback_input.py
+              tests/proxy_unit_tests/test_custom_logger_s3_gcs.py
+              tests/proxy_unit_tests/test_proxy_custom_logger.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: logging-misc
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_reject_logging.py
+              tests/proxy_unit_tests/test_audit_logs_proxy.py
+              tests/proxy_unit_tests/test_search_api_logging.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- db-and-spend: isolate the 170s schema-migration test ----
+          # test_db_schema_migration.py has exactly one test, and that test
+          # is mostly waiting on `prisma migrate deploy` / `prisma migrate
+          # diff` subprocesses (~170s). It does no CPU-bound Python work
+          # inside the test. Running with workers=0 (serial, no xdist)
+          # skips the 4-worker cold-start cost we'd otherwise pay for a
+          # single test, saving ~4 minutes of wall-clock.
+          - test-group: schema-migration
+            test-path: "tests/proxy_unit_tests/test_db_schema_migration.py"
+            workers: 0
+            dist: loadscope
+            timeout: 15
+          - test-group: db-and-spend
+            test-path: >-
+              tests/proxy_unit_tests/test_prisma_client_backoff_retry.py
+              tests/proxy_unit_tests/test_db_schema_changes.py
+              tests/proxy_unit_tests/test_e2e_pod_lock_manager.py
+              tests/proxy_unit_tests/test_skills_db.py
+              tests/proxy_unit_tests/test_update_daily_tag_spend.py
+              tests/proxy_unit_tests/test_update_spend.py
+              tests/proxy_unit_tests/test_proxy_encrypt_decrypt.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          # ---- guardrails + budget + hooks: split into 2 ----
+          - test-group: guardrails-hooks
+            test-path: >-
+              tests/proxy_unit_tests/test_proxy_setting_guardrails.py
+              tests/proxy_unit_tests/test_banned_keyword_list.py
+              tests/proxy_unit_tests/test_unit_test_proxy_hooks.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+          - test-group: budgets
+            test-path: >-
+              tests/proxy_unit_tests/test_default_end_user_budget_simple.py
+              tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py
+              tests/proxy_unit_tests/test_zero_cost_model_budget_bypass.py
+            workers: 4
+            dist: loadscope
+            timeout: 15
+
+          - test-group: endpoints-and-responses
+            test-path: >-
+              tests/proxy_unit_tests/test_blog_posts_endpoint.py
+              tests/proxy_unit_tests/test_models_fallback_endpoint.py
+              tests/proxy_unit_tests/test_google_endpoint_routing.py
+              tests/proxy_unit_tests/test_google_gemini_proxy_request.py
+              tests/proxy_unit_tests/test_gemini_agents_endpoints.py
+              tests/proxy_unit_tests/test_get_favicon.py
+              tests/proxy_unit_tests/test_get_image.py
+              tests/proxy_unit_tests/test_ui_path_detection.py
+              tests/proxy_unit_tests/test_prompt_test_endpoint.py
+              tests/proxy_unit_tests/test_check_batch_cost.py
+              tests/proxy_unit_tests/test_check_responses_cost.py
+              tests/proxy_unit_tests/test_response_polling_handler.py
+              tests/proxy_unit_tests/test_response_polling_pre_call_checks.py
+              tests/proxy_unit_tests/test_realtime_cache.py
+              tests/proxy_unit_tests/test_proxy_exception_mapping.py
+              tests/proxy_unit_tests/test_custom_tokenizer_bug.py
+              tests/proxy_unit_tests/test_model_response_typing
+            workers: 4
+            dist: loadscope
+            timeout: 15
     uses: ./.github/workflows/_test-unit-services-base.yml
     with:
       test-path: ${{ matrix.test-path }}
       workers: ${{ matrix.workers }}
       reruns: 2
       timeout-minutes: ${{ matrix.timeout }}
-      enable-redis: false
       enable-postgres: true
+      dist: ${{ matrix.dist }}
       artifact-name: proxy-db-${{ matrix.test-group }}
-    secrets:
-      DATABASE_URL: ${{ secrets.DATABASE_URL }}
-      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test-unit-proxy-endpoints.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy API Endpoints"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read
@@ -32,6 +36,8 @@ jobs:
         tests/test_litellm/proxy/health_endpoints
         tests/test_litellm/proxy/public_endpoints
         tests/test_litellm/proxy/prompts
+        tests/test_litellm/proxy/rag_endpoints
+        tests/test_litellm/proxy/realtime_endpoints
         tests/test_litellm/proxy/ui_crud_endpoints
       workers: 2
       reruns: 2

.github/workflows/test-unit-proxy-infra.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Infrastructure"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-proxy-legacy.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Legacy Tests"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read
@@ -24,7 +28,7 @@ jobs:
           - name: "key-generation"
             path: "tests/proxy_unit_tests/test_[k-o]*.py"
           - name: "proxy-config"
-            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
           - name: "proxy-server"
             path: "tests/proxy_unit_tests/test_proxy_server.py"
           - name: "proxy-server-extras"
@@ -50,43 +54,36 @@ jobs:
         with:
           python-version: "3.12"
 
-      - name: Install Poetry
-        run: pip install 'poetry==2.3.2'
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
 
-      - name: Cache Poetry dependencies
+      - name: Cache uv dependencies
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
-            ~/.cache/pypoetry
-            ~/.cache/pip
+            ~/.cache/uv
             .venv
-          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
           restore-keys: |
-            ${{ runner.os }}-poetry-
+            ${{ runner.os }}-uv-
 
       - name: Install dependencies
         run: |
-          poetry config virtualenvs.in-project true
-          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
-          poetry run pip install google-genai==1.22.0 \
-            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
-
-      - name: Setup litellm-enterprise
-        run: |
-          poetry run pip install --force-reinstall --no-deps -e enterprise/
+          uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
       - name: Generate Prisma client
         env:
           PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
-          poetry run pip install nodejs-wheel-binaries==24.13.1
-          poetry run prisma generate --schema litellm/proxy/schema.prisma
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Run tests - ${{ matrix.test-group.name }}
         env:
           TEST_PATH: ${{ matrix.test-group.path }}
         run: |
-          poetry run pytest ${TEST_PATH} \
+          uv run --no-sync pytest ${TEST_PATH} \
             --tb=short -vv \
             --maxfail=10 \
             -n 2 \

.github/workflows/test-unit-responses-caching-types.yml

@@ -2,7 +2,11 @@ name: "Unit Tests: Responses, Caching & Types"
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/test-unit-security.yml

@@ -1,9 +1,11 @@
 name: "Unit Tests: Security"
 
-# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
+# Kept push-only (was previously required by DATABASE_URL secret scoping;
+# now the postgres credentials are ephemeral localhost values but the
+# push-trigger stays to match the proxy-db workflow cadence).
 on:
   push:
-    branches: [main, "litellm_*"]
+    branches: [main, "litellm_**"]
 
 permissions:
   contents: read
@@ -22,10 +24,5 @@ jobs:
       workers: 1
       reruns: 2
       timeout-minutes: 20
-      enable-redis: false
       enable-postgres: true
       artifact-name: security
-    secrets:
-      DATABASE_URL: ${{ secrets.DATABASE_URL }}
-      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
-      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test_server_root_path.yml

@@ -4,12 +4,16 @@ permissions:
 
 on:
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_branch
+      - "litellm_**"
 
 jobs:
   test-server-root-path:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
 
     strategy:
       matrix:
@@ -21,6 +25,12 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Free up disk space
+        run: |
+          sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/share/boost
+          sudo apt-get clean
+          df -h /
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12
 

.github/workflows/update_release.py

@@ -1,54 +0,0 @@
-import os
-import requests
-from datetime import datetime
-
-# GitHub API endpoints
-GITHUB_API_URL = "https://api.github.com"
-REPO_OWNER = "BerriAI"
-REPO_NAME = "litellm"
-
-# GitHub personal access token (required for uploading release assets)
-GITHUB_ACCESS_TOKEN = os.environ.get("GITHUB_ACCESS_TOKEN")
-
-# Headers for GitHub API requests
-headers = {
-    "Accept": "application/vnd.github+json",
-    "Authorization": f"Bearer {GITHUB_ACCESS_TOKEN}",
-    "X-GitHub-Api-Version": "2022-11-28",
-}
-
-# Get the latest release
-releases_url = f"{GITHUB_API_URL}/repos/{REPO_OWNER}/{REPO_NAME}/releases/latest"
-response = requests.get(releases_url, headers=headers)
-latest_release = response.json()
-print("Latest release:", latest_release)
-
-# Upload an asset to the latest release
-upload_url = latest_release["upload_url"].split("{?")[0]
-asset_name = "results_stats.csv"
-asset_path = os.path.join(os.getcwd(), asset_name)
-print("upload_url:", upload_url)
-
-with open(asset_path, "rb") as asset_file:
-    asset_data = asset_file.read()
-
-upload_payload = {
-    "name": asset_name,
-    "label": "Load test results",
-    "created_at": datetime.utcnow().isoformat() + "Z",
-}
-
-upload_headers = headers.copy()
-upload_headers["Content-Type"] = "application/octet-stream"
-
-upload_response = requests.post(
-    upload_url,
-    headers=upload_headers,
-    data=asset_data,
-    params=upload_payload,
-)
-
-if upload_response.status_code == 201:
-    print(f"Asset '{asset_name}' uploaded successfully to the latest release.")
-else:
-    print(f"Failed to upload asset. Response: {upload_response.text}")

.gitignore

@@ -90,7 +90,6 @@ test.py
 litellm_config.yaml
 !.github/observatory/litellm_config.yaml
 .cursor
-.vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*
 update_model_cost_map.py
 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
@@ -100,4 +99,25 @@ STABILIZATION_TODO.md
 **/test-results
 **/playwright-report
 **/*.storageState.json
-**/coverage
+**/coverage
+test-config
+
+# ---------- Terraform ----------
+# Provider binaries + module cache — regenerated by `terraform init`.
+**/.terraform/
+# State files often contain secrets (DB passwords, API keys snapshotted from
+# data sources). Keep state in a remote backend, never in git.
+*.tfstate
+*.tfstate.*
+*.tfstate.backup
+# Plan files can also contain sensitive values (variables in plaintext).
+*.tfplan
+# User-specific variable inputs — example files (terraform.tfvars.example) are
+# tracked because they end in .example, which doesn't match the glob below.
+*.tfvars
+*.auto.tfvars
+crash.log
+crash.*.log
+# .terraform.lock.hcl is intentionally NOT ignored — it pins provider versions
+# and should be committed.
+.vscode

.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

AGENTS.md

@@ -23,10 +23,11 @@ LiteLLM is a unified interface for 100+ LLMs that:
 
 ### Key Directories
 - `tests/` - Comprehensive test suites
-- `docs/my-website/` - Documentation website
 - `ui/litellm-dashboard/` - Admin dashboard UI
 - `enterprise/` - Enterprise-specific features
 
+Documentation lives in the separate [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs) repository and is served at [docs.litellm.ai](https://docs.litellm.ai).
+
 ## DEVELOPMENT GUIDELINES
 
 ### MAKING CODE CHANGES
@@ -51,7 +52,9 @@ LiteLLM is a unified interface for 100+ LLMs that:
 
 ### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
 
-1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes**
+1. **Always use `antd` for new UI components — Tremor is DEPRECATED**
+   - We are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file.
+   - Use `antd` equivalents: `Tag` for labels, plain `<span>`/`<div>` with Tailwind classes (or `Typography.Text`) for text, `Card` from `antd`, etc. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
    - The only exception is the Tremor Table component and its required Tremor Table sub components.
 
 2. **Use Common Components as much as possible**:
@@ -121,7 +124,7 @@ LiteLLM supports MCP for agent workflows:
 
 ## RUNNING SCRIPTS
 
-Use `poetry run python script.py` to run Python scripts in the project environment (for non-test files).
+Use `uv run python script.py` to run Python scripts in the project environment (for non-test files).
 
 ## GITHUB TEMPLATES
 
@@ -216,8 +219,8 @@ When opening issues or pull requests, follow these templates:
 
 ## HELPFUL RESOURCES
 
-- Main documentation: https://docs.litellm.ai/
-- Provider-specific docs in `docs/my-website/docs/providers/`
+- Main documentation: https://docs.litellm.ai/ (source: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs))
+- Provider-specific docs: https://docs.litellm.ai/docs/providers/
 - Admin UI for testing proxy features
 
 ## WHEN IN DOUBT
@@ -232,16 +235,33 @@ When opening issues or pull requests, follow these templates:
 
 ### Environment
 
-- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
+- uv is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
 - Python 3.12, Node 22 are pre-installed.
-- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`.
+- The project virtual environment lives under `.venv/`.
 
 ### Running the proxy server
 
-Start the proxy with a config file:
+Create a minimal config file and start the proxy:
+
+```yaml
+# config.yaml
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake-model
+      api_key: fake-key
+      api_base: https://fake-api.example.com
+
+general_settings:
+  master_key: sk-1234
+
+litellm_settings:
+  drop_params: True
+  telemetry: False
+```
 
 ```bash
-poetry run litellm --config dev_config.yaml --port 4000
+uv run litellm --config config.yaml --port 4000
 ```
 
 The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
@@ -250,17 +270,16 @@ The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot
 
 See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
 
-- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
-- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
+- `uv sync --group proxy-dev --extra proxy` installs the Prisma and proxy-side test dependencies used by the standard local workflow.
 - The `--timeout` pytest flag is NOT available; don't pass it.
-- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
-- **Before committing, always run `poetry run black .` to format your code.** Black formatting is enforced in CI.
-- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
+- Unit tests: `uv run pytest tests/test_litellm/ -x -vv -n 4`
+- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI.
+- If `uv sync` fails because the lockfile is outdated, run `uv lock` and retry.
 
 ### Lint
 
 ```bash
-cd litellm && poetry run ruff check .
+cd litellm && uv run ruff check .
 ```
 
 Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
@@ -271,4 +290,4 @@ Ruff is the primary fast linter. For the full lint suite (including mypy, black,
 - The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
 - SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
 - Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
-- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`
+- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`

CLAUDE.md

@@ -2,12 +2,16 @@
 
 This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
 
+## Documentation
+
+Documentation lives in a separate repository: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs). It is served at [docs.litellm.ai](https://docs.litellm.ai). Do not create or edit documentation files in this repository — open doc PRs against `BerriAI/litellm-docs` instead.
+
 ## Development Commands
 
 ### Installation
 - `make install-dev` - Install core development dependencies
 - `make install-proxy-dev` - Install proxy development dependencies with full feature set
-- `make install-test-deps` - Install all test dependencies
+- `make install-test-deps` - Install the full local test environment and generate the Prisma client
 
 ### Testing
 - `make test` - Run all tests
@@ -20,14 +24,14 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 - `make format` - Apply Black code formatting
 - `make lint-ruff` - Run Ruff linting only
 - `make lint-mypy` - Run MyPy type checking only
-- **Before committing, always run `poetry run black .` to format your code.** Black formatting is enforced in CI.
+- **Before committing, always run `uv run black .` to format your code.** Black formatting is enforced in CI.
 
 ### Single Test Files
-- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
-- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
+- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file
+- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
 
 ### Running Scripts
-- `poetry run python script.py` - Run Python scripts (use for non-test files)
+- `uv run python script.py` - Run Python scripts (use for non-test files)
 
 ### GitHub Issue & PR Templates
 When contributing to the project, use the appropriate templates:
@@ -109,7 +113,11 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 ### UI / Backend Consistency
 - When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
 
+### UI Component Library
+- **Always use `antd` for new UI components** — we are migrating off of `@tremor/react`. Do not introduce new `Badge`, `Text`, `Card`, `Grid`, `Title`, or other imports from `@tremor/react` in any new or modified file. Use `antd` equivalents: `Tag` for labels, `Typography.Text` / `Typography.Title` / `Typography.Paragraph` for textual content (avoid plain text-only `<span>`, `<p>`, `<h*>` when Typography fits), and `Card` from `antd`. Note that `antd` has no `"yellow"` Tag color — use `"gold"` for amber/yellow.
+
 ### MCP OAuth / OpenAPI Transport Mapping
+- **`available_on_public_internet: false` with `delegate_auth_to_upstream: true` (oauth2, interactive — not `client_credentials`)** — LiteLLM still allows the anonymous upstream PKCE path (no proxy API key for `/authorize` and matching MCP routes). The internal-only flag mainly affects other surfaces (e.g. IP-based discovery). Rely on the upstream IdP and network policy; the dashboard shows a warning when both are set, and the proxy logs a warning when the server is loaded from config or the database.
 - `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
 - FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
 - When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
@@ -139,7 +147,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
 - **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
 - **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
-- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
+- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
 
 ### Setup Wizard (`litellm/setup_wizard.py`)
 - The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).
@@ -151,6 +159,14 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Optional features enabled via environment variables
 - Separate licensing and authentication for enterprise features
 
+### CI Supply-Chain Safety
+- **Never pipe a remote script into a shell** (`curl ... | bash`, `wget ... | sh`). Download the artifact to a file, verify its SHA-256 checksum, then install.
+- **Pin every external tool to a specific version** with a full URL (not `latest` or `stable`). Unversioned downloads silently change under you.
+- **Verify checksums for all downloaded binaries.** Use the provider's official `.sha256` / `.sha256sum` sidecar file when available; otherwise compute and hardcode the digest.
+- **Prefer reusable CircleCI commands** (`commands:` section) so a tool is installed and verified in exactly one place, then referenced everywhere with `- install_<tool>` or `- wait_for_service`.
+- **Don't add tools just because they were there before.** Audit whether an external dependency is still needed. If it can be replaced with a shell one-liner or a tool already in the image, remove it.
+- These rules apply to every download in CI: binaries, install scripts, language version managers, package repos. No exceptions.
+
 ### HTTP Client Cache Safety
 - **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
 

CONTRIBUTING.md

@@ -122,9 +122,17 @@ Run all unit tests (uses parallel execution for speed):
 make test-unit
 ```
 
+If you're running broader test suites, proxy tests, or anything that touches PostgreSQL-backed fixtures/plugins, install the full local test environment first:
+
+```bash
+make install-test-deps
+```
+
+This syncs the locked test environment used across the repo, including `psycopg` v3 plus `psycopg-binary` (used by `pytest-postgresql`), `psycopg2-binary` (used by some proxy E2E tests), and a generated Prisma client for DB-backed proxy tests, so pytest startup matches CI without manual package installs.
+
 Run specific test files:
 ```bash
-poetry run pytest tests/test_litellm/test_your_file.py -v
+uv run pytest tests/test_litellm/test_your_file.py -v
 ```
 
 ### Running Linting and Formatting Checks
@@ -185,7 +193,7 @@ Run `make help` to see all available commands:
 make help                       # Show all available commands
 make install-dev               # Install development dependencies
 make install-proxy-dev         # Install proxy development dependencies
-make install-test-deps         # Install test dependencies (for running tests)
+make install-test-deps         # Install the full local test environment
 make format                    # Apply Black code formatting
 make format-check              # Check Black formatting (matches CI)
 make lint                      # Run all linting checks
@@ -247,7 +255,7 @@ To run the proxy server locally:
 make install-proxy-dev
 
 # Start the proxy server
-poetry run litellm --config your_config.yaml
+uv run litellm --config your_config.yaml
 ```
 
 ### Docker Development
@@ -332,4 +340,4 @@ Looking for ideas? Check out:
 - 🧪 Test coverage improvements
 - 🔌 New LLM provider integrations
 
-Thank you for contributing to LiteLLM! 🚀 
+Thank you for contributing to LiteLLM! 🚀 

Dockerfile

@@ -1,142 +1,101 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
+
+FROM $UV_IMAGE AS uvbin
 
 # Builder stage
 FROM $LITELLM_BUILD_IMAGE AS builder
 
-# Set the working directory to /app
 WORKDIR /app
-
 USER root
 
-# Install build dependencies
-RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev
+COPY --from=uvbin /uv /usr/local/bin/uv
+COPY --from=uvbin /uvx /usr/local/bin/uvx
 
-RUN python -m pip install build==1.4.2
+RUN apk add --no-cache \
+    bash \
+    gcc \
+    python3 \
+    python3-dev \
+    openssl \
+    openssl-dev \
+    nodejs \
+    npm \
+    libsndfile
 
-# Copy the current directory contents into the container at /app
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
+    UV_LINK_MODE=copy \
+    PATH="/app/.venv/bin:${PATH}"
+
+# Copy dependency metadata first for layer caching
+COPY pyproject.toml uv.lock ./
+COPY enterprise/pyproject.toml enterprise/
+COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
+
+# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
+RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
+    --extra proxy \
+    --extra proxy-runtime \
+    --extra extra_proxy \
+    --extra semantic-router \
+    --python python3
+
+# Copy full source tree
 COPY . .
 
-# Build Admin UI
-# Convert Windows line endings to Unix and make executable
+# Build Admin UI before final sync
 RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
-# Build the package
-RUN rm -rf dist/* && python -m build
+# Install project and workspace packages (fast - deps already cached)
+RUN uv sync --frozen --no-default-groups --no-editable \
+    --extra proxy \
+    --extra proxy-runtime \
+    --extra extra_proxy \
+    --extra semantic-router \
+    --python python3
 
-# There should be only one wheel file now, assume the build only creates one
-RUN ls -1 dist/*.whl | head -1
+RUN prisma generate --schema=./schema.prisma
 
-# Install the package
-RUN pip install dist/*.whl
-
-# install dependencies as wheels
-RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
-
-# ensure pyjwt is used, not jwt
-RUN pip uninstall jwt -y
-RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.12.0 --no-cache-dir
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
+    sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
-# Ensure runtime stage runs as root
 USER root
 
-# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
-    # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
-    # levels inside its dependency tree. `npm install -g <pkg>` only creates a
-    # SEPARATE global package, it does NOT replace npm's internal copies.
-    # We must find and replace EVERY copy inside npm's directory.
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
+    npm install -g npm@11.14.0 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
     GLOBAL="$(npm root -g)" && \
-    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    for pkg in tar glob @isaacs/brace-expansion brace-expansion minimatch diff picomatch; do \
+        name="${pkg##*/}"; \
+        find "$GLOBAL/npm" -type d -name "$name" -path "*/node_modules/$pkg" | while read d; do \
+            rm -rf "$d" && cp -rL "$GLOBAL/$pkg" "$d"; \
+        done; \
     done && \
-    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done && \
-    # SECURITY FIX: patch npm's own package.json metadata so scanners see the
-    # actual installed versions instead of the stale declared dependencies.
-    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
     npm cache clean --force && \
-    # Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
-    # no longer visible to image scanners.  The globally installed npm@latest
-    # at /usr/local/lib/node_modules/npm/ remains fully functional.
     { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
-# Copy the current directory contents into the container at /app
-COPY . .
-RUN ls -la /app
+ENV PATH="/app/.venv/bin:${PATH}"
 
-# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
-COPY --from=builder /app/dist/*.whl .
-COPY --from=builder /wheels/ /wheels/
+COPY --from=builder /app /app
+# Prisma binaries live in $HOME/.cache (default prisma-python location),
+# which is /root/.cache here. Copy only the Prisma subdirs — copying the
+# whole /root/.cache drags in the uv build cache (~660 MB, includes a
+# setuptools wheel that surfaces as a CVE finding even though it's not
+# on the runtime sys.path).
+COPY --from=builder /root/.cache/prisma /root/.cache/prisma
+COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
 
-# Install the built wheel using pip; again using a wildcard if it's the only file
-RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels
-
-# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
-RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
-    if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
-
-# Remove test files and keys from dependencies
-RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
-    find /usr/lib -type d -path "*/tornado/test" -delete
-
-# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
-# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
-# Patch every copy of tar, glob, and brace-expansion inside that tree.
-RUN GLOBAL="$(npm root -g)" && \
-    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
-    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-    done && \
-    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done
-
-# Install semantic_router and aurelio-sdk using script
-# Convert Windows line endings to Unix and make executable
-RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
-
-# Generate prisma client using the correct schema
-RUN prisma generate --schema=./litellm/proxy/schema.prisma
-# Convert Windows line endings to Unix for entrypoint scripts
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
-RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
+RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
+    find /app/.venv -type d -path "*/tornado/test" -delete
 
 EXPOSE 4000/tcp
 
-RUN apk add --no-cache supervisor
-COPY docker/supervisord.conf /etc/supervisord.conf
-
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
-
-# Append "--detailed_debug" to the end of CMD to view detailed debug logs
 CMD ["--port", "4000"]

GEMINI.md

@@ -22,11 +22,11 @@ This file provides guidance to Gemini when working with code in this repository.
 - `make lint-mypy` - Run MyPy type checking only
 
 ### Single Test Files
-- `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
-- `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
+- `uv run pytest tests/path/to/test_file.py -v` - Run specific test file
+- `uv run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
 
 ### Running Scripts
-- `poetry run python script.py` - Run Python scripts (use for non-test files)
+- `uv run python script.py` - Run Python scripts (use for non-test files)
 
 ### GitHub Issue & PR Templates
 When contributing to the project, use the appropriate templates:
@@ -105,4 +105,4 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 ### Enterprise Features
 - Enterprise-specific code in `enterprise/` directory
 - Optional features enabled via environment variables
-- Separate licensing and authentication for enterprise features
+- Separate licensing and authentication for enterprise features

Makefile

@@ -15,7 +15,7 @@ help:
 	@echo "  make install-proxy-dev  - Install proxy development dependencies"
 	@echo "  make install-dev-ci     - Install dev dependencies (CI-compatible, pins OpenAI)"
 	@echo "  make install-proxy-dev-ci - Install proxy dev dependencies (CI-compatible)"
-	@echo "  make install-test-deps  - Install test dependencies"
+	@echo "  make install-test-deps  - Install the full local test environment"
 	@echo "  make install-helm-unittest - Install helm unittest plugin"
 	@echo "  make format             - Apply Black code formatting"
 	@echo "  make format-check       - Check Black code formatting (matches CI)"
@@ -40,49 +40,44 @@ help:
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
 
-# Keep PIP simple for edge cases:
-PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip")
+UV := uv
+UV_RUN := $(UV) run --no-sync
 
 # Show info
 info:
-	@echo "PIP: $(PIP)"
+	@echo "UV: $(UV)"
 
 # Installation targets
 install-dev:
-	poetry install --with dev
+	$(UV) sync --frozen
 
 install-proxy-dev:
-	poetry install --with dev,proxy-dev --extras proxy
+	$(UV) sync --frozen --group proxy-dev --extra proxy
 
 # CI-compatible installations (matches GitHub workflows exactly)
 install-dev-ci:
-	$(PIP) install openai==2.8.0
-	poetry install --with dev
-	$(PIP) install openai==2.8.0
+	$(UV) sync --frozen
 
 install-proxy-dev-ci:
-	poetry install --with dev,proxy-dev --extras proxy
-	$(PIP) install openai==2.8.0
+	$(UV) sync --frozen --group proxy-dev --extra proxy
 
 install-test-deps: install-proxy-dev
-	poetry run $(PIP) install "pytest-retry==1.6.3"
-	poetry run $(PIP) install pytest-xdist
-	poetry run $(PIP) install openapi-core
-	cd enterprise && poetry run $(PIP) install -e . && cd ..
+	$(UV) sync --frozen --all-groups --all-extras
+	$(UV_RUN) prisma generate --schema litellm/proxy/schema.prisma
 
 install-helm-unittest:
 	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
 
 # Formatting
 format: install-dev
-	cd litellm && poetry run black . && cd ..
+	cd litellm && $(UV_RUN) black . && cd ..
 
 format-check: install-dev
-	cd litellm && poetry run black --check . && cd ..
+	cd litellm && $(UV_RUN) black --check . && cd ..
 
 # Linting targets
 lint-ruff: install-dev
-	cd litellm && poetry run ruff check . && cd ..
+	cd litellm && $(UV_RUN) ruff check . && cd ..
 
 # faster linter for developing ...
 # inspiration from:
@@ -96,37 +91,36 @@ lint-format-changed: install-dev
 			$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
 			print "$$file:$$start:1-$$end:999\n"; \
 		}' | \
-	while read range; do \
-		file="$${range%%:*}"; \
-		lines="$${range#*:}"; \
-		echo "Formatting $$file (lines $$lines)"; \
-		poetry run ruff format --range "$$lines" "$$file"; \
-	done
+		while read range; do \
+			file="$${range%%:*}"; \
+			lines="$${range#*:}"; \
+			echo "Formatting $$file (lines $$lines)"; \
+			$(UV_RUN) ruff format --range "$$lines" "$$file"; \
+		done
 
 lint-ruff-dev: install-dev
 	@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
 	cd litellm && \
-	(poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \
-	poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
+	($(UV_RUN) ruff check . --output-format=pylint || true) > "$$tmpfile" && \
+	$(UV_RUN) diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
 	cd .. ; \
 	rm -f "$$tmpfile"
 
 lint-ruff-FULL-dev: install-dev
 	@files=$$(git diff --name-only origin/main -- '*.py'); \
-	if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \
+	if [ -n "$$files" ]; then echo "$$files" | xargs $(UV_RUN) ruff check; \
 	else echo "No changed .py files to check."; fi
 
 lint-mypy: install-dev
-	poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML
-	cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
+	cd litellm && $(UV_RUN) mypy . --ignore-missing-imports && cd ..
 
 lint-black: format-check
 
 check-circular-imports: install-dev
-	cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
+	cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
 
 check-import-safety: install-dev
-	@poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+	@$(UV_RUN) python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
 
 # Combined linting (matches test-linting.yml workflow)
 lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
@@ -135,46 +129,46 @@ lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safet
 lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
 
 # Testing targets
-test:
-	poetry run pytest tests/
+test: install-test-deps
+	$(UV_RUN) pytest tests/
 
 test-unit: install-test-deps
-	poetry run pytest tests/test_litellm -x -vv -n 4
+	$(UV_RUN) pytest tests/test_litellm -x -vv -n 4
 
 # Matrix test targets (matching CI workflow groups)
 test-unit-llms: install-test-deps
-	poetry run pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-guardrails: install-test-deps
-	poetry run pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-core: install-test-deps
-	poetry run pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
 
 test-unit-proxy-misc: install-test-deps
-	poetry run pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
 
 test-unit-integrations: install-test-deps
-	poetry run pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
 
 test-unit-core-utils: install-test-deps
-	poetry run pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
+	$(UV_RUN) pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
 
 test-unit-other: install-test-deps
-	poetry run pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
 
 test-unit-root: install-test-deps
-	poetry run pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
+	$(UV_RUN) pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
 
 # Proxy unit tests (tests/proxy_unit_tests split alphabetically)
 test-proxy-unit-a: install-test-deps
-	poetry run pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
+	$(UV_RUN) pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
 
 test-proxy-unit-b: install-test-deps
-	poetry run pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
+	$(UV_RUN) pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
 
-test-integration:
-	poetry run pytest tests/ -k "not test_litellm"
+test-integration: install-test-deps
+	$(UV_RUN) pytest tests/ -k "not test_litellm"
 
 test-unit-helm: install-helm-unittest
 	helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
@@ -188,6 +182,9 @@ test-llm-translation-single: install-test-deps
 	@echo "Running single LLM translation test file..."
 	@if [ -z "$(FILE)" ]; then echo "Usage: make test-llm-translation-single FILE=test_filename.py"; exit 1; fi
 	@mkdir -p test-results
-	poetry run pytest tests/llm_translation/$(FILE) \
+	$(UV_RUN) pytest tests/llm_translation/$(FILE) \
 		--junitxml=test-results/junit.xml \
 		-v --tb=short --maxfail=100 --timeout=300
+
+test-llm-translation-flush-vcr-cache:
+	$(UV_RUN) python tests/_flush_vcr_cache.py

README.md

@@ -12,7 +12,7 @@
         </a>
         </p>
     </p>
-<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://litellm.ai/enterprise"target="_blank">Enterprise Tier</a> | <a href="https://litellm.ai/" target="_blank">Website</a></h4>
+<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://litellm.ai/enterprise"target="_blank">Enterprise Tier</a> | <a href="https://www.litellm.ai/ai-gateway" target="_blank">Website</a></h4>
 <h4 align="center">
     <a href="https://pypi.org/project/litellm/" target="_blank">
         <img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
@@ -39,8 +39,45 @@
 
 <img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
 
+---
 
-## Use LiteLLM for
+## What is LiteLLM
+
+LiteLLM is an open source AI Gateway that gives you a single, unified interface to call 100+ LLM providers — OpenAI, Anthropic, Gemini, Bedrock, Azure, and more — using the OpenAI format.
+
+Use it as a **Python SDK** for direct library integration, or deploy the **AI Gateway (Proxy Server)** as a centralized service for your team or organization.
+
+[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy) <br>
+[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
+
+---
+
+## Why LiteLLM
+
+Managing LLM calls across providers gets complicated fast — different SDKs, auth patterns, request formats, and error types for every model. LiteLLM removes that friction:
+
+- **Unified API** — one interface for 100+ LLMs, no provider-specific SDK juggling
+- **Drop-in OpenAI compatibility** — swap providers without rewriting your code
+- **Production-ready gateway** — virtual keys, spend tracking, guardrails, load balancing, and an admin dashboard out of the box
+- **8ms P95 latency** at 1k RPS ([benchmarks](https://docs.litellm.ai/docs/benchmarks))
+
+### OSS Adopters
+
+<table>
+  <tr>
+    <td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
+    <td><img height="60" alt="image" src="https://github.com/user-attachments/assets/436fca71-988b-40bb-b5fe-8450c80fdbd0" /></td>
+    <td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
+    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/3db0ae72-0843-4005-a56d-bba1dde2193d" /></td>
+    <td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
+    <td><h2>Netflix</h2></td>
+    <td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
+  </tr>
+</table>
+
+---
+
+## Features
 
 <details open>
 <summary><b>LLMs</b> - Call 100+ LLMs (Python SDK + AI Gateway)</summary>
@@ -50,7 +87,7 @@
 ### Python SDK
 
 ```shell
-pip install litellm
+uv add litellm
 ```
 
 ```python
@@ -72,7 +109,7 @@ response = completion(model="anthropic/claude-sonnet-4-20250514", messages=[{"ro
 [**Getting Started - E2E Tutorial**](https://docs.litellm.ai/docs/proxy/docker_quick_start) - Setup virtual keys, make your first request
 
 ```shell
-pip install 'litellm[proxy]'
+uv tool install 'litellm[proxy]'
 litellm --model gpt-4o
 ```
 
@@ -223,63 +260,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 
 </details>
 
----
-
-## How to use LiteLLM
-
-You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
-
-<table style={{width: '100%', tableLayout: 'fixed'}}>
-<thead>
-<tr>
-<th style={{width: '14%'}}></th>
-<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
-<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
-</tr>
-</thead>
-<tbody>
-<tr>
-<td style={{width: '14%'}}><strong>Use Case</strong></td>
-<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
-<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
-</tr>
-<tr>
-<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
-<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
-<td style={{width: '43%'}}>Developers building LLM projects</td>
-</tr>
-<tr>
-<td style={{width: '14%'}}><strong>Key Features</strong></td>
-<td style={{width: '43%'}}>Centralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and management</td>
-<td style={{width: '43%'}}>Direct Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a>, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
-</tr>
-</tbody>
-</table>
-
-LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
-
-[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy) <br>
-[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
-
-**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
-
-Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
-
-## OSS Adopters 
-
-<table>
-  <tr>
-    <td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
-    <td><img height="60" alt="image" src="https://github.com/user-attachments/assets/436fca71-988b-40bb-b5fe-8450c80fdbd0" /></td>
-    <td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
-    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
-    <td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
-    <td><h2>Netflix</h2></td>
-    <td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
-  </tr>
-</table>
-
-## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
+### Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
 
 | Provider                                                                            | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
 |-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
@@ -311,7 +292,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
 | [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ |  |
-| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |
 | [Databricks (`databricks`)](https://docs.litellm.ai/docs/providers/databricks) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [DataRobot (`datarobot`)](https://docs.litellm.ai/docs/providers/datarobot) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ |  |  | ✅ |  |  |  |  |
@@ -386,25 +367,62 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature
 
 [**Read the Docs**](https://docs.litellm.ai/docs/)
 
-## Run in Developer mode
-### Services
+---
+
+## Get Started
+
+You can use LiteLLM through either the Proxy Server or Python SDK. Both give you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
+
+<table style={{width: '100%', tableLayout: 'fixed'}}>
+<thead>
+<tr>
+<th style={{width: '14%'}}></th>
+<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
+<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td style={{width: '14%'}}><strong>Use Case</strong></td>
+<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
+<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
+</tr>
+<tr>
+<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
+<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
+<td style={{width: '43%'}}>Developers building LLM projects</td>
+</tr>
+<tr>
+<td style={{width: '14%'}}><strong>Key Features</strong></td>
+<td style={{width: '43%'}}>Centralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and management</td>
+<td style={{width: '43%'}}>Direct Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a>, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
+</tr>
+</tbody>
+</table>
+
+**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
+
+Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
+
+### Run in Developer Mode
+#### Services
 1. Setup .env file in root
 2. Run dependant services `docker-compose up db prometheus`
 
-### Backend
+#### Backend
 1. (In root) create virtual environment `python -m venv .venv`
 2. Activate virtual environment `source .venv/bin/activate`
-3. Install dependencies `pip install -e ".[all]"`
-4. `pip install prisma`
+3. Install dependencies `uv sync --all-extras --group proxy-dev`
+4. `uv run prisma generate`
 5. `prisma generate`
 6. Start proxy backend `python litellm/proxy/proxy_cli.py`
 
-### Frontend
+#### Frontend
 1. Navigate to `ui/litellm-dashboard`
 2. Install dependencies `npm install`
 3. Run `npm run dev` to start the dashboard
 
-# Verify Docker Image Signatures
+### Verify Docker Image Signatures
 
 All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
 
@@ -430,6 +448,8 @@ cosign verify \
 
 Replace `<release-tag>` with the version you are deploying (e.g. `v1.83.0-stable`).
 
+---
+
 # Enterprise
 For companies that need better security, user management and professional support
 
@@ -450,7 +470,7 @@ We welcome contributions to LiteLLM! Whether you're fixing bugs, adding features
 
 ## Quick Start for Contributors
 
-This requires poetry to be installed.
+This requires uv to be installed.
 
 ```bash
 git clone https://github.com/BerriAI/litellm.git
@@ -464,6 +484,8 @@ make format-check   # Check formatting only
 
 For detailed contributing guidelines, see [CONTRIBUTING.md](CONTRIBUTING.md).
 
+> **📖 Contributing to documentation?** The LiteLLM docs have moved to a separate repository: [BerriAI/litellm-docs](https://github.com/BerriAI/litellm-docs). Please open doc PRs there. Docs are served at [docs.litellm.ai](https://docs.litellm.ai).
+
 ## Code Quality / Linting
 
 LiteLLM follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html).
@@ -486,10 +508,6 @@ All these checks must pass before your PR can be merged.
 - [Community Slack 💭](https://www.litellm.ai/support)
 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
 
-# Why did we build this
-
-- **Need for simplicity**: Our code started to get extremely complicated managing & translating calls between Azure, OpenAI and Cohere.
-
 # Contributors
 
 <!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
@@ -504,4 +522,3 @@ All these checks must pass before your PR can be merged.
 <a href="https://github.com/BerriAI/litellm/graphs/contributors">
   <img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
 </a>
-

backend/Dockerfile

@@ -0,0 +1,83 @@
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
+
+FROM $UV_IMAGE AS uvbin
+
+# ---------- Builder ----------
+FROM $LITELLM_BUILD_IMAGE AS builder
+
+WORKDIR /app
+USER root
+
+COPY --from=uvbin /uv /uvx /usr/local/bin/
+
+RUN apk add --no-cache bash gcc python3 python3-dev openssl openssl-dev libsndfile
+
+# UV_COMPILE_BYTECODE=1   precompiles .pyc at install time → faster cold start.
+# UV_LINK_MODE=copy       avoids hardlink warnings when uv installs from a
+#                         BuildKit cache mount (different filesystem).
+# UV_PYTHON_DOWNLOADS=0   force uv to use the apk-installed CPython instead of
+#                         silently pulling a managed interpreter.
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
+    UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=0 \
+    PATH="/app/.venv/bin:${PATH}"
+
+# Stage 1 — install dependencies only.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=enterprise/pyproject.toml,target=enterprise/pyproject.toml \
+    --mount=type=bind,source=litellm-proxy-extras/pyproject.toml,target=litellm-proxy-extras/pyproject.toml \
+    uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
+        --extra proxy \
+        --extra proxy-runtime \
+        --extra extra_proxy \
+        --extra semantic-router \
+        --python python3
+
+# Stage 2 — copy source and install the project + workspace members.
+COPY . .
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-default-groups --no-editable \
+        --extra proxy \
+        --extra proxy-runtime \
+        --extra extra_proxy \
+        --extra semantic-router \
+        --python python3
+
+RUN mkdir -p /home/nonroot && \
+    HOME=/home/nonroot prisma generate --schema=./schema.prisma && \
+    chown -R nonroot:nonroot /home/nonroot/.cache
+
+# ---------- Runtime ----------
+FROM $LITELLM_RUNTIME_IMAGE AS runtime
+
+USER root
+
+RUN apk add --no-cache bash openssl tzdata python3 libsndfile libatomic
+
+# wolfi-base ships an unprivileged `nonroot` account (UID/GID 65532) with
+# /home/nonroot. We run the backend as that user
+WORKDIR /app
+ENV HOME=/home/nonroot \
+    PATH="/app/.venv/bin:${PATH}" \
+    PYTHONPATH="/app" \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+COPY --from=builder --chown=nonroot:nonroot /app /app
+COPY --from=builder --chown=nonroot:nonroot /home/nonroot/.cache /home/nonroot/.cache
+
+RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
+    find /app/.venv -type d -path "*/tornado/test" -delete
+
+USER nonroot
+
+EXPOSE 4001/tcp
+
+ENTRYPOINT ["uvicorn", "backend.main:app"]
+CMD ["--host", "0.0.0.0", "--port", "4001"]

backend/main.py

@@ -0,0 +1,51 @@
+"""UI backend entrypoint.
+
+Reuses the existing FastAPI app from `litellm.proxy.proxy_server` and trims its
+route table to just the management/admin surface used by the dashboard. Purely
+additive — no existing module is modified.
+
+Run with:
+    uvicorn backend.main:app --host 0.0.0.0 --port 4001
+"""
+
+from contextlib import asynccontextmanager
+
+from fastapi.routing import Mount
+
+# See gateway/main.py for why we assemble DATABASE_URL(s) here before
+# importing proxy_server.
+from litellm.proxy.db.db_url_settings import DatabaseURLSettings
+
+DatabaseURLSettings.from_env().apply_to_env()
+
+from litellm.proxy.proxy_server import app
+
+from backend.routes.allowlist import BACKEND_EXACT_PATHS, BACKEND_PATH_PREFIXES
+
+
+def _is_backend_route(route) -> bool:
+    """Keep the route on the backend if its path is in the management surface."""
+    path = getattr(route, "path", None)
+    if path is None:
+        return False
+    if isinstance(route, Mount):
+        # Static UI mounts are served by the dedicated UI container, not here.
+        return False
+    if path in BACKEND_EXACT_PATHS:
+        return True
+    return any(path.startswith(prefix) for prefix in BACKEND_PATH_PREFIXES)
+
+
+# See gateway/main.py for why the trim runs inside the lifespan instead of at
+# module scope.
+_proxy_lifespan = app.router.lifespan_context
+
+
+@asynccontextmanager
+async def _backend_lifespan(app_):
+    async with _proxy_lifespan(app_):
+        app_.router.routes = [r for r in app_.router.routes if _is_backend_route(r)]
+        yield
+
+
+app.router.lifespan_context = _backend_lifespan

backend/routes/allowlist.py

@@ -0,0 +1,135 @@
+"""Path allowlist for the UI backend (control plane) component.
+
+The backend exposes management/admin endpoints consumed by the UI: keys, users,
+teams, orgs, customers, budgets, tags, workflows, model management, spend &
+analytics, settings (router/cache/cost-tracking/fallbacks), SSO/onboarding,
+audit logs, debug, enterprise admin, and UI bootstrap helpers (logo, favicon,
+.well-known config).
+
+Anything LLM data-plane is dropped — those run on the gateway component.
+"""
+
+BACKEND_PATH_PREFIXES: tuple[str, ...] = (
+    # Identity / access
+    "/key/",
+    "/v2/key/",
+    "/user/",
+    "/v2/user/",
+    "/team/",
+    "/v2/team/",
+    "/organization/",
+    "/customer/",
+    "/end_user/",
+    "/sso/",
+    "/login",
+    "/v2/login",
+    "/v3/login",
+    "/logout",
+    "/token",
+    "/onboarding/",
+    "/audit",
+    "/oauth/",
+    "/invitation/",
+    "/jwt/",
+    # Models & routing config
+    "/model/",
+    "/v1/model/info",
+    "/v2/model/",
+    "/model_group",
+    "/model_access_group/",
+    "/model_hub/",
+    "/v1/access_group",
+    "/access_group/",
+    "/router/",
+    "/router_settings",
+    "/adaptive_router/",
+    "/fallback",
+    "/fallbacks",
+    "/cache_settings",
+    "/cost_tracking",
+    "/cost/",
+    "/credentials",
+    "/credential",
+    "/provider/budgets",
+    # Tools / agents (registry & policy admin)
+    "/v1/tool/",
+    "/v1/agents",
+    # Guardrails admin
+    "/v2/guardrails/",
+    # MCP server admin + BYOK OAuth flow (UI-initiated) + dynamic per-server endpoints
+    "/v1/mcp/",
+    "/test/",
+    "/{mcp_server_name}/",
+    # Budgets / tags / workflows / memory mgmt
+    "/budget/",
+    "/tag/",
+    "/workflow/",
+    "/v1/workflows/",
+    "/project/",
+    "/memory/",
+    "/mcp/",
+    # Spend / analytics
+    "/spend/",
+    "/analytics/",
+    "/global/",
+    "/user_agent",
+    "/usage/",
+    "/daily/",
+    # CloudZero cost-export admin (init / settings / export / dry-run / delete)
+    "/cloudzero/",
+    # Caching admin
+    "/cache/",
+    "/caching/",
+    # Callbacks / hooks
+    "/active/callbacks",
+    "/callbacks",
+    "/team_callback",
+    # Alerting / email / IP allowlist
+    "/alerting/",
+    "/email/",
+    "/add/allowed_ip",
+    "/delete/allowed_ip",
+    "/get/",
+    # Enterprise admin
+    "/enterprise/",
+    # Debug / config / profiling
+    "/debug/",
+    "/config/",
+    "/memory-usage-in-mem-cache",
+    "/otel-spans",
+    "/lazy/",
+    "/in_product_nudges",
+    # Admin reload / schedule
+    "/reload/",
+    "/schedule/",
+    "/settings",
+    "/update/",
+    "/upload/",
+    # Dev / admin utilities
+    "/utils/",
+    # UI bootstrap helpers (assets the dashboard fetches)
+    "/get_logo_url",
+    "/get_image",
+    "/get_favicon",
+    "/.well-known/",
+    "/litellm/.well-known/",
+    "/ui_discovery/",
+    "/ui-config",
+    "/sso_settings",
+    "/public/",
+    "/robots.txt",
+    # Health (k8s probes)
+    "/health",
+)
+
+BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
+    {
+        "/",
+        "/routes",
+        "/openapi.json",
+        "/docs",
+        "/docs/oauth2-redirect",
+        "/redoc",
+        "/fallback/login",
+    }
+)

ci_cd/run_migration.py

@@ -1,23 +1,234 @@
+import argparse
 import os
-import subprocess
-from pathlib import Path
-from datetime import datetime
-import testing.postgresql
+import re
 import shutil
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+
+import testing.postgresql
 
 
-def create_migration(migration_name: str = None):
+DESTRUCTIVE_PATTERN = re.compile(r"\bDROP\s+(COLUMN|TABLE|INDEX)\b", re.IGNORECASE)
+DEFAULT_BASE_BRANCH = "litellm_internal_staging"
+
+
+def _find_destructive_statements(sql: str) -> list:
+    """Return SQL lines containing DROP COLUMN, DROP TABLE, or DROP INDEX."""
+    return [
+        line.strip() for line in sql.splitlines() if DESTRUCTIVE_PATTERN.search(line)
+    ]
+
+
+def _print_freshness_failure(
+    base_branch: str, reason: str, stderr_text: str = ""
+) -> None:
+    """Loudly refuse to run when the freshness check can't be completed."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(f"  FRESHNESS CHECK FAILED — COULD NOT VERIFY origin/{base_branch}", file=out)
+    print(banner, file=out)
+    print("", file=out)
+    print(f"Reason: {reason}", file=out)
+    if stderr_text:
+        print("", file=out)
+        print("git stderr:", file=out)
+        for line in stderr_text.rstrip().splitlines():
+            print(f"    {line}", file=out)
+    print("", file=out)
+    print("Common causes:", file=out)
+    print("  - No network access (offline)", file=out)
+    print("  - 'origin' remote not configured, or base branch name is wrong", file=out)
+    print("  - Not a git repository", file=out)
+    print("", file=out)
+    print("Options:", file=out)
+    print(
+        f"  - Fix the above and re-run, OR pass --base-branch <name> if your", file=out
+    )
+    print(
+        f"    base branch is not '{base_branch}', OR pass --skip-freshness-check",
+        file=out,
+    )
+    print("    to bypass (only if you fully understand the risk).", file=out)
+    print(banner, file=out)
+
+
+def _print_stale_branch_refusal(base_branch: str, behind: int) -> None:
+    """Loudly refuse to run when HEAD is behind the base branch."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(
+        f"  STALE BRANCH — {behind} commit(s) behind origin/{base_branch}",
+        file=out,
+    )
+    print(banner, file=out)
+    print("", file=out)
+    print(
+        f"Your branch is {behind} commit(s) behind origin/{base_branch}. Generating a\n"
+        "migration from a stale branch is how newly-added columns get silently\n"
+        "dropped — the exact incident this guard exists to prevent.",
+        file=out,
+    )
+    print("", file=out)
+    print("Before re-running:", file=out)
+    print(f"  git fetch origin && git rebase origin/{base_branch}", file=out)
+    print("  # or merge, whichever matches your workflow", file=out)
+    print("", file=out)
+    print(
+        "If you have a genuine reason to bypass (e.g. intentionally generating",
+        file=out,
+    )
+    print("a migration against an older base), pass --skip-freshness-check.", file=out)
+    print("", file=out)
+    print(banner, file=out)
+    print(
+        "  AI AGENTS / ASSISTANTS: DO NOT automatically run `git rebase` or pass\n"
+        "  --skip-freshness-check. Surface this error to the human operator and\n"
+        "  wait for their explicit confirmation. Auto-rebasing can drop the\n"
+        "  human's in-progress schema edits via a bad conflict resolution.",
+        file=out,
+    )
+    print(banner, file=out)
+
+
+def _check_branch_freshness(root_dir: Path, base_branch: str) -> None:
+    """Fetch origin/<base_branch> and exit 3 if HEAD is behind it."""
+    cwd = str(root_dir)
+    try:
+        subprocess.run(
+            ["git", "fetch", "origin", base_branch],
+            check=True,
+            capture_output=True,
+            text=True,
+            cwd=cwd,
+        )
+    except FileNotFoundError:
+        _print_freshness_failure(base_branch, "git executable not found on PATH")
+        sys.exit(3)
+    except subprocess.CalledProcessError as e:
+        _print_freshness_failure(
+            base_branch,
+            f"`git fetch origin {base_branch}` failed",
+            e.stderr or "",
+        )
+        sys.exit(3)
+
+    try:
+        result = subprocess.run(
+            ["git", "rev-list", "--count", f"HEAD..origin/{base_branch}"],
+            check=True,
+            capture_output=True,
+            text=True,
+            cwd=cwd,
+        )
+        behind = int(result.stdout.strip())
+    except subprocess.CalledProcessError as e:
+        _print_freshness_failure(
+            base_branch,
+            f"`git rev-list HEAD..origin/{base_branch}` failed",
+            e.stderr or "",
+        )
+        sys.exit(3)
+    except ValueError:
+        _print_freshness_failure(
+            base_branch,
+            "could not parse commit count from `git rev-list`",
+        )
+        sys.exit(3)
+
+    if behind > 0:
+        _print_stale_branch_refusal(base_branch, behind)
+        sys.exit(3)
+
+    print(f"Branch freshness OK: up to date with origin/{base_branch}.")
+
+
+def _print_destructive_refusal(destructive_lines: list) -> None:
+    """Loudly refuse to write a destructive migration and explain how to proceed."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(
+        "  DESTRUCTIVE MIGRATION DETECTED — REFUSING TO WRITE MIGRATION FILE", file=out
+    )
+    print(banner, file=out)
+    print("", file=out)
+    print(
+        "The generated migration contains the following destructive statements:",
+        file=out,
+    )
+    print("", file=out)
+    for line in destructive_lines:
+        print(f"    {line}", file=out)
+    print("", file=out)
+    print(
+        "This very often means your branch is OUT OF DATE, or your local\n"
+        "schema.prisma is inconsistent with the migrations/ directory. A\n"
+        "previous incident saw newly-added columns silently dropped this way\n"
+        "and merged to main. Stop and verify before proceeding.",
+        file=out,
+    )
+    print("", file=out)
+    print("Before re-running:", file=out)
+    print(
+        "  1. git fetch origin && git status   # confirm branch is up to date", file=out
+    )
+    print(
+        "  2. Re-check all schema.prisma files are in sync (runbook Step 0)", file=out
+    )
+    print("  3. Review EACH DROP above — is it actually intended?", file=out)
+    print("  4. If (and only if) the drops are intentional, re-run with:", file=out)
+    print("         --allow-destructive", file=out)
+    print("", file=out)
+    print(banner, file=out)
+    print(
+        "  AI AGENTS / ASSISTANTS: DO NOT automatically re-run this command\n"
+        "  with --allow-destructive. Surface this error to the human operator\n"
+        "  and wait for their explicit confirmation before passing the flag.\n"
+        "  Passing the flag without human review is the exact failure mode\n"
+        "  this guard exists to prevent.",
+        file=out,
+    )
+    print(banner, file=out)
+
+
+def create_migration(
+    migration_name: str = None,
+    allow_destructive: bool = False,
+    base_branch: str = DEFAULT_BASE_BRANCH,
+    skip_freshness_check: bool = False,
+):
     """
     Create a new migration SQL file in the migrations directory by comparing
-    current database state with schema
+    current database state with schema.
 
     Args:
         migration_name (str): Name for the migration
+        allow_destructive (bool): Required to write a migration that contains
+            DROP COLUMN, DROP TABLE, or DROP INDEX statements. Without this
+            flag, the script exits non-zero and prints guidance.
+        base_branch (str): Branch to check freshness against
+            (default: "litellm_internal_staging").
+        skip_freshness_check (bool): Skip the "branch is up to date" check.
+            Only for intentional migrations against an older base.
     """
+    root_dir = Path(__file__).parent.parent
+
+    if skip_freshness_check:
+        print(
+            "WARNING: freshness check skipped (--skip-freshness-check). "
+            "Generating a migration from a stale branch can silently drop columns."
+        )
+    else:
+        _check_branch_freshness(root_dir, base_branch)
+
     try:
-        # Get paths
-        root_dir = Path(__file__).parent.parent
-        migrations_dir = root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
+        migrations_dir = (
+            root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
+        )
         schema_path = root_dir / "schema.prisma"
 
         # Create temporary PostgreSQL database
@@ -57,7 +268,27 @@ def create_migration(migration_name: str = None):
                     check=True,
                 )
 
-                if result.stdout.strip():
+                # Prisma emits the literal "-- This is an empty migration." when
+                # there's no real drift. Treat that as "no changes".
+                diff_sql = result.stdout
+                stripped = diff_sql.strip()
+                is_empty_diff = (
+                    not stripped or stripped == "-- This is an empty migration."
+                )
+
+                if not is_empty_diff:
+                    destructive_lines = _find_destructive_statements(diff_sql)
+                    if destructive_lines and not allow_destructive:
+                        _print_destructive_refusal(destructive_lines)
+                        sys.exit(2)
+                    if destructive_lines and allow_destructive:
+                        print(
+                            "WARNING: writing destructive migration "
+                            "(--allow-destructive passed). Statements:"
+                        )
+                        for line in destructive_lines:
+                            print(f"    {line}")
+
                     # Generate timestamp and create migration directory
                     timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
                     migration_name = migration_name or "unnamed_migration"
@@ -66,7 +297,7 @@ def create_migration(migration_name: str = None):
 
                     # Write the SQL to migration.sql
                     migration_file = migration_dir / "migration.sql"
-                    migration_file.write_text(result.stdout)
+                    migration_file.write_text(diff_sql)
 
                     print(f"Created migration in {migration_dir}")
                     return True
@@ -88,8 +319,48 @@ def create_migration(migration_name: str = None):
 
 
 if __name__ == "__main__":
-    # If running directly, can optionally pass migration name as argument
-    import sys
-
-    migration_name = sys.argv[1] if len(sys.argv) > 1 else None
-    create_migration(migration_name)
+    parser = argparse.ArgumentParser(
+        description=(
+            "Generate a Prisma migration by diffing the temp DB "
+            "(existing migrations applied) against schema.prisma."
+        )
+    )
+    parser.add_argument(
+        "migration_name",
+        nargs="?",
+        default=None,
+        help="Name for the migration (used in the generated directory name).",
+    )
+    parser.add_argument(
+        "--allow-destructive",
+        action="store_true",
+        help=(
+            "Required to write a migration that contains DROP COLUMN, "
+            "DROP TABLE, or DROP INDEX. Without this flag, destructive "
+            "diffs are refused."
+        ),
+    )
+    parser.add_argument(
+        "--base-branch",
+        default=DEFAULT_BASE_BRANCH,
+        help=(
+            f"Branch to check freshness against (default: {DEFAULT_BASE_BRANCH}). "
+            "The script fetches origin/<base-branch> and refuses to run if HEAD "
+            "is behind it."
+        ),
+    )
+    parser.add_argument(
+        "--skip-freshness-check",
+        action="store_true",
+        help=(
+            "Bypass the 'branch is up to date' check. Only for intentional "
+            "migrations against an older base. Pairs poorly with automation."
+        ),
+    )
+    args = parser.parse_args()
+    create_migration(
+        args.migration_name,
+        allow_destructive=args.allow_destructive,
+        base_branch=args.base_branch,
+        skip_freshness_check=args.skip_freshness_check,
+    )

codecov.yaml

@@ -1,3 +1,18 @@
+codecov:
+  require_ci_to_pass: false # post coverage status even if CI has unrelated failures
+  notify:
+    wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
+
+# Uploads are flagged per workflow/shard (GHA) or "circleci". carryforward makes
+# a re-upload of a flag replace its prior session instead of accumulating a
+# conflicting one, and lets a commit reuse a flag from its parent when that flag
+# was not re-uploaded. Required because the same commit can receive the
+# push-triggered workflows more than once (re-runs / branches cut at the same
+# SHA); flagless overlapping sessions made Codecov drop the largest files.
+flag_management:
+  default_rules:
+    carryforward: true
+
 component_management:
   individual_components:
     - component_id: "Router"
@@ -28,7 +43,7 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 1% # at maximum allow project coverage to drop by 1%
+        threshold: 0% # do not allow project coverage to drop
     patch:
       default:
         target: auto

cookbook/anthropic_agent_sdk/agent_with_mcp.py

@@ -24,24 +24,26 @@ async def interactive_chat_with_mcp():
     Interactive CLI chat with the agent and MCP server
     """
     config = Config()
-    
+
     # Configure Anthropic SDK to point to LiteLLM gateway
     litellm_base_url = setup_litellm_env(config)
-    
+
     # Fetch available models from proxy
-    available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
-    
+    available_models = await fetch_available_models(
+        litellm_base_url, config.LITELLM_API_KEY
+    )
+
     current_model = config.LITELLM_MODEL
-    
+
     # MCP server configuration
     mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
     use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
-    
+
     if not use_mcp:
         print("⚠️  MCP disabled via USE_MCP=false")
-    
+
     print_header(litellm_base_url, current_model, has_mcp=use_mcp)
-    
+
     while True:
         # Configure agent options
         if use_mcp:
@@ -58,7 +60,7 @@ async def interactive_chat_with_mcp():
                             "url": mcp_server_url,
                             "headers": {
                                 "Authorization": f"Bearer {config.LITELLM_API_KEY}"
-                            }
+                            },
                         }
                     },
                 )
@@ -78,12 +80,12 @@ async def interactive_chat_with_mcp():
                 model=current_model,
                 max_turns=50,
             )
-        
+
         # Create agent client
         try:
             async with ClaudeSDKClient(options=options) as client:
                 conversation_active = True
-                
+
                 while conversation_active:
                     # Get user input
                     try:
@@ -91,34 +93,36 @@ async def interactive_chat_with_mcp():
                     except (EOFError, KeyboardInterrupt):
                         print("\n\n👋 Goodbye!")
                         return
-                    
+
                     # Handle commands
-                    if user_input.lower() in ['quit', 'exit']:
+                    if user_input.lower() in ["quit", "exit"]:
                         print("\n👋 Goodbye!")
                         return
-                    
-                    if user_input.lower() == 'clear':
+
+                    if user_input.lower() == "clear":
                         print("\n🔄 Starting new conversation...\n")
                         conversation_active = False
                         continue
-                    
-                    if user_input.lower() == 'models':
+
+                    if user_input.lower() == "models":
                         handle_model_list(available_models, current_model)
                         continue
-                    
-                    if user_input.lower() == 'model':
-                        new_model, should_restart = handle_model_switch(available_models, current_model)
+
+                    if user_input.lower() == "model":
+                        new_model, should_restart = handle_model_switch(
+                            available_models, current_model
+                        )
                         if should_restart:
                             current_model = new_model
                             conversation_active = False
                         continue
-                    
+
                     if not user_input:
                         continue
-                    
+
                     # Stream response from agent
                     await stream_response(client, user_input)
-        
+
         except Exception as e:
             print(f"\n❌ Error creating agent client: {e}")
             print("This might be an MCP configuration issue. Try running without MCP:")

cookbook/anthropic_agent_sdk/common.py

@@ -8,13 +8,13 @@ import httpx
 
 class Config:
     """Configuration for LiteLLM Gateway connection"""
-    
+
     # LiteLLM proxy URL (default to local instance)
     LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
-    
+
     # LiteLLM API key (master key or virtual key)
     LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
-    
+
     # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
     LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
 
@@ -28,7 +28,7 @@ async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
             response = await client.get(
                 f"{base_url}/models",
                 headers={"Authorization": f"Bearer {api_key}"},
-                timeout=10.0
+                timeout=10.0,
             )
             response.raise_for_status()
             data = response.json()
@@ -50,7 +50,7 @@ def setup_litellm_env(config: Config):
     """
     Configure environment variables to point Agent SDK to LiteLLM
     """
-    litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
+    litellm_base_url = config.LITELLM_PROXY_URL.rstrip("/")
     os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
     os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
     return litellm_base_url
@@ -87,10 +87,12 @@ def handle_model_list(available_models: list[str], current_model: str):
         print(f"  {marker} {i}. {model}")
 
 
-def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
+def handle_model_switch(
+    available_models: list[str], current_model: str
+) -> tuple[str, bool]:
     """
     Handle model switching
-    
+
     Returns:
         tuple: (new_model, should_restart_conversation)
     """
@@ -98,7 +100,7 @@ def handle_model_switch(available_models: list[str], current_model: str) -> tupl
     for i, model in enumerate(available_models, 1):
         marker = "✓" if model == current_model else " "
         print(f"  {marker} {i}. {model}")
-    
+
     try:
         choice = input("\nEnter number (or press Enter to cancel): ").strip()
         if choice:
@@ -112,7 +114,7 @@ def handle_model_switch(available_models: list[str], current_model: str) -> tupl
                 print("❌ Invalid choice")
     except (ValueError, IndexError):
         print("❌ Invalid input")
-    
+
     return current_model, False
 
 
@@ -120,41 +122,43 @@ async def stream_response(client, user_input: str):
     """
     Stream response from the agent
     """
-    print("\n🤖 Assistant: ", end='', flush=True)
-    
+    print("\n🤖 Assistant: ", end="", flush=True)
+
     try:
         await client.query(user_input)
-        
+
         # Show loading indicator
-        print("⏳ thinking...", end='', flush=True)
-        
+        print("⏳ thinking...", end="", flush=True)
+
         # Stream the response
         first_chunk = True
         async for msg in client.receive_response():
             # Clear loading indicator on first message
             if first_chunk:
-                print("\r🤖 Assistant: ", end='', flush=True)
+                print("\r🤖 Assistant: ", end="", flush=True)
                 first_chunk = False
-            
+
             # Handle different message types
-            if hasattr(msg, 'type'):
-                if msg.type == 'content_block_delta':
+            if hasattr(msg, "type"):
+                if msg.type == "content_block_delta":
                     # Streaming text delta
-                    if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
-                        print(msg.delta.text, end='', flush=True)
-                elif msg.type == 'content_block_start':
+                    if hasattr(msg, "delta") and hasattr(msg.delta, "text"):
+                        print(msg.delta.text, end="", flush=True)
+                elif msg.type == "content_block_start":
                     # Start of content block
-                    if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
-                        print(msg.content_block.text, end='', flush=True)
-            
+                    if hasattr(msg, "content_block") and hasattr(
+                        msg.content_block, "text"
+                    ):
+                        print(msg.content_block.text, end="", flush=True)
+
             # Fallback to original content handling
-            if hasattr(msg, 'content'):
+            if hasattr(msg, "content"):
                 for content_block in msg.content:
-                    if hasattr(content_block, 'text'):
-                        print(content_block.text, end='', flush=True)
-        
+                    if hasattr(content_block, "text"):
+                        print(content_block.text, end="", flush=True)
+
         print()  # New line after response
-        
+
     except Exception as e:
         print(f"\r\n❌ Error: {e}")
         print("Please check your LiteLLM gateway is running and configured correctly.")

cookbook/anthropic_agent_sdk/main.py

@@ -24,17 +24,19 @@ async def interactive_chat():
     Interactive CLI chat with the agent
     """
     config = Config()
-    
+
     # Configure Anthropic SDK to point to LiteLLM gateway
     litellm_base_url = setup_litellm_env(config)
-    
+
     # Fetch available models from proxy
-    available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
-    
+    available_models = await fetch_available_models(
+        litellm_base_url, config.LITELLM_API_KEY
+    )
+
     current_model = config.LITELLM_MODEL
-    
+
     print_header(litellm_base_url, current_model)
-    
+
     while True:
         # Configure agent options for each conversation
         options = ClaudeAgentOptions(
@@ -42,11 +44,11 @@ async def interactive_chat():
             model=current_model,
             max_turns=50,
         )
-        
+
         # Create agent client
         async with ClaudeSDKClient(options=options) as client:
             conversation_active = True
-            
+
             while conversation_active:
                 # Get user input
                 try:
@@ -54,31 +56,33 @@ async def interactive_chat():
                 except (EOFError, KeyboardInterrupt):
                     print("\n\n👋 Goodbye!")
                     return
-                
+
                 # Handle commands
-                if user_input.lower() in ['quit', 'exit']:
+                if user_input.lower() in ["quit", "exit"]:
                     print("\n👋 Goodbye!")
                     return
-                
-                if user_input.lower() == 'clear':
+
+                if user_input.lower() == "clear":
                     print("\n🔄 Starting new conversation...\n")
                     conversation_active = False
                     continue
-                
-                if user_input.lower() == 'models':
+
+                if user_input.lower() == "models":
                     handle_model_list(available_models, current_model)
                     continue
-                
-                if user_input.lower() == 'model':
-                    new_model, should_restart = handle_model_switch(available_models, current_model)
+
+                if user_input.lower() == "model":
+                    new_model, should_restart = handle_model_switch(
+                        available_models, current_model
+                    )
                     if should_restart:
                         current_model = new_model
                         conversation_active = False
                     continue
-                
+
                 if not user_input:
                     continue
-                
+
                 # Stream response from agent
                 await stream_response(client, user_input)
 

cookbook/litellm-ollama-docker-image/requirements.txt

@@ -1 +1 @@
-litellm==1.61.15
+litellm==1.83.14

cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py

@@ -11,15 +11,15 @@ BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
 batch_input_file = client.files.create(
     file=open("./bedrock_batch_completions.jsonl", "rb"),
     purpose="batch",
-    extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
+    extra_body={"target_model_names": BEDROCK_BATCH_MODEL},
 )
 print(batch_input_file)
 
 # Create batch
-batch = client.batches.create( 
+batch = client.batches.create(
     input_file_id=batch_input_file.id,
     endpoint="/v1/chat/completions",
     completion_window="24h",
     metadata={"description": "Test batch job"},
 )
-print(batch)
+print(batch)

cookbook/litellm_proxy_server/cli_token_usage.py

@@ -8,6 +8,7 @@ in your Python scripts after running `litellm-proxy login`.
 
 from textwrap import indent
 import litellm
+
 LITELLM_BASE_URL = "http://localhost:4000/"
 
 
@@ -15,38 +16,38 @@ def main():
     """Using CLI token with LiteLLM SDK"""
     print("🚀 Using CLI Token with LiteLLM SDK")
     print("=" * 40)
-    #litellm._turn_on_debug()
-    
+    # litellm._turn_on_debug()
+
     # Get the CLI token
     api_key = litellm.get_litellm_gateway_api_key()
-    
+
     if not api_key:
         print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
         return
-    
+
     print("✅ Found CLI token.")
 
     available_models = litellm.get_valid_models(
         check_provider_endpoint=True,
         custom_llm_provider="litellm_proxy",
         api_key=api_key,
-        api_base=LITELLM_BASE_URL
+        api_base=LITELLM_BASE_URL,
     )
-    
+
     print("✅ Available models:")
     if available_models:
         for i, model in enumerate(available_models, 1):
             print(f"   {i:2d}. {model}")
     else:
         print("   No models available")
-    
+
     # Use with LiteLLM
     try:
         response = litellm.completion(
             model="litellm_proxy/gemini/gemini-2.5-flash",
             messages=[{"role": "user", "content": "Hello from CLI token!"}],
             api_key=api_key,
-            base_url=LITELLM_BASE_URL
+            base_url=LITELLM_BASE_URL,
         )
         print(f"✅ LLM Response: {response.model_dump_json(indent=4)}")
     except Exception as e:
@@ -55,7 +56,7 @@ def main():
 
 if __name__ == "__main__":
     main()
-    
+
     print("\n💡 Tips:")
     print("1. Run 'litellm-proxy login' to authenticate first")
     print("2. Replace 'https://your-proxy.com' with your actual proxy URL")

cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py

@@ -3,11 +3,12 @@ Use LiteLLM Proxy MCP Gateway to call MCP tools.
 
 When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
 """
+
 import openai
 
 client = openai.OpenAI(
-    api_key="sk-1234", # paste your litellm proxy api key here
-    base_url="http://localhost:4000" # paste your litellm proxy base url here
+    api_key="sk-1234",  # paste your litellm proxy api key here
+    base_url="http://localhost:4000",  # paste your litellm proxy base url here
 )
 print("Making API request to Responses API with MCP tools")
 
@@ -17,7 +18,7 @@ response = client.responses.create(
         {
             "role": "user",
             "content": "give me TLDR of what BerriAI/litellm repo is about",
-            "type": "message"
+            "type": "message",
         }
     ],
     tools=[
@@ -25,11 +26,11 @@ response = client.responses.create(
             "type": "mcp",
             "server_label": "litellm",
             "server_url": "litellm_proxy",
-            "require_approval": "never"
+            "require_approval": "never",
         }
     ],
     stream=True,
-    tool_choice="required"
+    tool_choice="required",
 )
 
 for chunk in response:

cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py

@@ -40,8 +40,10 @@ class InMemorySecretManager(CustomSecretManager):
     ) -> Optional[str]:
         """Read secret synchronously"""
         from litellm._logging import verbose_proxy_logger
-        
-        verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}")
+
+        verbose_proxy_logger.info(
+            f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}"
+        )
         value = self.secrets.get(secret_name)
         verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: READ SECRET: {value}")
         return value
@@ -76,4 +78,3 @@ class InMemorySecretManager(CustomSecretManager):
             del self.secrets[secret_name]
             return {"status": "deleted", "secret_name": secret_name}
         return {"status": "not_found", "secret_name": secret_name}
-

cookbook/livekit_agent_sdk/main.py

@@ -5,6 +5,7 @@ This example shows how to use LiveKit's xAI realtime plugin through LiteLLM prox
 LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI, 
 and Azure realtime APIs without changing your agent code.
 """
+
 import asyncio
 import json
 import os
@@ -23,71 +24,79 @@ async def run_voice_agent():
     2. Sends a user message
     3. Streams back the response
     """
-    
+
     url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}"
     headers = {"Authorization": f"Bearer {API_KEY}"}
-    
+
     print(f"🎙️  Connecting to voice agent...")
     print(f"   Model: {MODEL}")
     print(f"   Proxy: {PROXY_URL}")
     print()
-    
+
     async with websockets.connect(url, additional_headers=headers) as ws:
         # Receive initial connection event
         initial = json.loads(await ws.recv())
         print(f"✅ Connected! Event: {initial['type']}\n")
-        
+
         # Get user input
         user_message = input("💬 Your message: ").strip()
         if not user_message:
             user_message = "Tell me a fun fact about AI!"
-        
+
         print(f"\n🤖 Sending to {MODEL}...\n")
-        
+
         # Send user message
-        await ws.send(json.dumps({
-            "type": "conversation.item.create",
-            "item": {
-                "type": "message",
-                "role": "user",
-                "content": [{"type": "input_text", "text": user_message}]
-            }
-        }))
-        
+        await ws.send(
+            json.dumps(
+                {
+                    "type": "conversation.item.create",
+                    "item": {
+                        "type": "message",
+                        "role": "user",
+                        "content": [{"type": "input_text", "text": user_message}],
+                    },
+                }
+            )
+        )
+
         # Request response
-        await ws.send(json.dumps({
-            "type": "response.create",
-            "response": {"modalities": ["text", "audio"]}
-        }))
-        
+        await ws.send(
+            json.dumps(
+                {
+                    "type": "response.create",
+                    "response": {"modalities": ["text", "audio"]},
+                }
+            )
+        )
+
         # Stream response
-        print("🎤 Response: ", end='', flush=True)
+        print("🎤 Response: ", end="", flush=True)
         transcript = []
-        
+
         try:
             while True:
                 msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
                 event = json.loads(msg)
-                
+
                 # Capture transcript deltas
-                if event['type'] == 'response.output_audio_transcript.delta':
-                    delta = event.get('delta', '')
+                if event["type"] == "response.output_audio_transcript.delta":
+                    delta = event.get("delta", "")
                     if delta:
-                        print(delta, end='', flush=True)
+                        print(delta, end="", flush=True)
                         transcript.append(delta)
-                
+
                 # Done when response completes
-                elif event['type'] == 'response.done':
+                elif event["type"] == "response.done":
                     break
-        
+
         except asyncio.TimeoutError:
             pass
-        
+
         print("\n")
-        
+
         if transcript:
             print(f"✅ Complete response: {''.join(transcript)}")
-        
+
         await ws.close()
 
 
@@ -97,7 +106,7 @@ def main():
     print("LiveKit xAI Voice Agent via LiteLLM Proxy")
     print("=" * 70)
     print()
-    
+
     try:
         asyncio.run(run_voice_agent())
     except KeyboardInterrupt:

cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md

@@ -9,6 +9,32 @@ This document provides comprehensive instructions for AI agents to generate rele
 3. **Previous Version Commit Hash** - To compare model pricing changes
 4. **Reference Release Notes** - Use recent stable releases (v1.76.3-stable, v1.77.2-stable) as templates for consistent formatting
 
+### Resolving Staging PRs
+
+The GitHub release page (e.g. `https://github.com/BerriAI/litellm/releases/tag/v1.83.3-stable`) does **not** list the real changelog directly. The "What's Changed" section contains **staging PRs** that each bundle many individual commits/PRs. For example:
+
+- `Litellm oss staging 03 14 2026 by @RheagalFire in #23686`
+- `Litellm ryan march 16 by @ryan-crabbe in #23822`
+
+To get the real changelog, you MUST click into each staging PR (e.g. `#23686`, `#23822`), open its **Commits** tab, and extract every underlying commit/PR (look for the `(#NNNNN)` suffix on commit titles). Those underlying PRs — not the staging PRs — are what get categorized in the release notes. Never treat a staging PR title as a single changelog entry.
+
+**IMPORTANT — staging PRs are not the complete source.** Some PRs land on the release branch *before* the staging PRs and are therefore not reachable via `gh api /pulls/<staging>/commits`. GitHub's auto-generated "What's Changed" on the release page also misses these. To catch every PR in the release, you MUST additionally walk the full git log range between the previous release's commit and this release's commit:
+
+```bash
+git fetch origin --tags
+git log <prev_release_commit>..<this_release_commit> --oneline | grep -oE '#[0-9]+' | sort -u
+```
+
+Union the PR set from the staging-PR walk with the PR set from `git log`. Any PR in `git log` but missing from your staging-expanded set is almost certainly a content PR that merged directly to the release branch — fetch its title/body with `gh pr view <N>` and categorize it. Do not trust the GH release body or the staging PRs alone as the authoritative list.
+
+**Sanity check for new contributors.** The GH release body's "New Contributors" list is a *floor*, not authoritative. For every PR author who appears in the release (including underlying PRs from staging and PRs found only via `git log`), verify whether they are a first-time contributor by running:
+
+```bash
+gh api "search/issues?q=is:pr+author:<login>+repo:BerriAI/litellm+is:merged&sort=created&order=asc" --jq '.items[0] | {n:.number, merged:.closed_at}'
+```
+
+If the author's earliest merged PR number matches a PR in this release window, they are a new contributor. If their earliest merged PR predates the previous release tag, they are not. Do not copy the GH release body's list blindly — it can both miss contributors (PRs that merged via an older dev branch) and falsely include contributors whose "first" PR in this window was not actually their first ever.
+
 ## Step-by-Step Process
 
 ### 1. Initial Setup and Analysis

cookbook/misc/test_responses_api.py

@@ -1,10 +1,9 @@
 import base64
 from openai import OpenAI
 import time
-client = OpenAI(
-    base_url="http://0.0.0.0:4001",
-    api_key="sk-1234"
-)
+
+client = OpenAI(base_url="http://0.0.0.0:4001", api_key="sk-1234")
+
 
 # Function to encode the image
 def encode_image(image_path):
@@ -25,7 +24,7 @@ response = client.responses.create(
         {
             "role": "user",
             "content": [
-                { "type": "input_text", "text": "what color is the image"},
+                {"type": "input_text", "text": "what color is the image"},
                 {
                     "type": "input_image",
                     "image_url": f"data:image/jpeg;base64,{base64_image}",
@@ -36,7 +35,6 @@ response = client.responses.create(
 )
 
 
-
 print(response.output_text)
 print("response1 id===", response.id)
 print("sleeping for 20 seconds...")
@@ -45,9 +43,7 @@ print("making follow up request for existing id")
 response2 = client.responses.create(
     model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
     previous_response_id=response.id,
-    input="ok, and what objects are in the image?"
+    input="ok, and what objects are in the image?",
 )
 
 print(response2.output_text)
-
-

cookbook/nova_sonic_realtime.py

@@ -52,11 +52,11 @@ class RealtimeClient:
     async def connect(self):
         """Connect to LiteLLM proxy realtime endpoint."""
         print(f"Connecting to {self.url}...")
-        
+
         headers = {}
         if self.api_key:
             headers["Authorization"] = f"Bearer {self.api_key}"
-        
+
         self.ws = await websockets.connect(
             self.url,
             additional_headers=headers,
@@ -175,7 +175,9 @@ class RealtimeClient:
 
         try:
             while self.is_active:
-                audio_data = self.input_stream.read(CHUNK_SIZE, exception_on_overflow=False)
+                audio_data = self.input_stream.read(
+                    CHUNK_SIZE, exception_on_overflow=False
+                )
                 await self.send_audio_chunk(audio_data)
                 await asyncio.sleep(0.01)  # Small delay to prevent overwhelming
         except Exception as e:
@@ -270,6 +272,7 @@ async def main():
     except Exception as e:
         print(f"\n❌ Error: {e}")
         import traceback
+
         traceback.print_exc()
     finally:
         await client.close()
@@ -281,7 +284,7 @@ if __name__ == "__main__":
     print("2. Bedrock is configured in proxy_server_config.yaml")
     print("3. AWS credentials are set")
     print()
-    
+
     try:
         asyncio.run(main())
     except KeyboardInterrupt:

cookbook/veo_video_generation.py

@@ -21,49 +21,45 @@ from typing import Optional
 
 class VeoVideoGenerator:
     """Complete Veo video generation client using LiteLLM proxy."""
-    
-    def __init__(self, base_url: str = "http://localhost:4000/gemini/v1beta", 
-                 api_key: str = "sk-1234"):
+
+    def __init__(
+        self,
+        base_url: str = "http://localhost:4000/gemini/v1beta",
+        api_key: str = "sk-1234",
+    ):
         """
         Initialize the Veo video generator.
-        
+
         Args:
             base_url: Base URL for the LiteLLM proxy with Gemini pass-through
             api_key: API key for LiteLLM proxy authentication
         """
         self.base_url = base_url
         self.api_key = api_key
-        self.headers = {
-            "x-goog-api-key": api_key,
-            "Content-Type": "application/json"
-        }
-    
+        self.headers = {"x-goog-api-key": api_key, "Content-Type": "application/json"}
+
     def generate_video(self, prompt: str) -> Optional[str]:
         """
         Initiate video generation with Veo.
-        
+
         Args:
             prompt: Text description of the video to generate
-            
+
         Returns:
             Operation name if successful, None otherwise
         """
         print(f"🎬 Generating video with prompt: '{prompt}'")
-        
+
         url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning"
-        payload = {
-            "instances": [{
-                "prompt": prompt
-            }]
-        }
-        
+        payload = {"instances": [{"prompt": prompt}]}
+
         try:
             response = requests.post(url, headers=self.headers, json=payload)
             response.raise_for_status()
-            
+
             data = response.json()
             operation_name = data.get("name")
-            
+
             if operation_name:
                 print(f"✅ Video generation started: {operation_name}")
                 return operation_name
@@ -71,58 +67,64 @@ class VeoVideoGenerator:
                 print("❌ No operation name returned")
                 print(f"Response: {json.dumps(data, indent=2)}")
                 return None
-                
+
         except requests.RequestException as e:
             print(f"❌ Failed to start video generation: {e}")
-            if hasattr(e, 'response') and e.response is not None:
+            if hasattr(e, "response") and e.response is not None:
                 try:
                     error_data = e.response.json()
                     print(f"Error details: {json.dumps(error_data, indent=2)}")
                 except:
                     print(f"Error response: {e.response.text}")
             return None
-    
-    def wait_for_completion(self, operation_name: str, max_wait_time: int = 600) -> Optional[str]:
+
+    def wait_for_completion(
+        self, operation_name: str, max_wait_time: int = 600
+    ) -> Optional[str]:
         """
         Poll operation status until video generation is complete.
-        
+
         Args:
             operation_name: Name of the operation to monitor
             max_wait_time: Maximum time to wait in seconds (default: 10 minutes)
-            
+
         Returns:
             Video URI if successful, None otherwise
         """
         print("⏳ Waiting for video generation to complete...")
-        
+
         operation_url = f"{self.base_url}/{operation_name}"
         start_time = time.time()
         poll_interval = 10  # Start with 10 seconds
-        
+
         while time.time() - start_time < max_wait_time:
             try:
-                print(f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)")
-                
+                print(
+                    f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)"
+                )
+
                 response = requests.get(operation_url, headers=self.headers)
                 response.raise_for_status()
-                
+
                 data = response.json()
-                
+
                 # Check for errors
                 if "error" in data:
                     print("❌ Error in video generation:")
                     print(json.dumps(data["error"], indent=2))
                     return None
-                
+
                 # Check if operation is complete
                 is_done = data.get("done", False)
-                
+
                 if is_done:
                     print("🎉 Video generation complete!")
-                    
+
                     try:
                         # Extract video URI from nested response
-                        video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
+                        video_uri = data["response"]["generateVideoResponse"][
+                            "generatedSamples"
+                        ][0]["video"]["uri"]
                         print(f"📹 Video URI: {video_uri}")
                         return video_uri
                     except KeyError as e:
@@ -130,64 +132,68 @@ class VeoVideoGenerator:
                         print("Full response:")
                         print(json.dumps(data, indent=2))
                         return None
-                
+
                 # Wait before next poll, with exponential backoff
                 time.sleep(poll_interval)
                 poll_interval = min(poll_interval * 1.2, 30)  # Cap at 30 seconds
-                
+
             except requests.RequestException as e:
                 print(f"❌ Error polling operation status: {e}")
                 time.sleep(poll_interval)
-        
+
         print(f"⏰ Timeout after {max_wait_time} seconds")
         return None
-    
-    def download_video(self, video_uri: str, output_filename: str = "generated_video.mp4") -> bool:
+
+    def download_video(
+        self, video_uri: str, output_filename: str = "generated_video.mp4"
+    ) -> bool:
         """
         Download the generated video file.
-        
+
         Args:
             video_uri: URI of the video to download (from Google's response)
             output_filename: Local filename to save the video
-            
+
         Returns:
             True if download successful, False otherwise
         """
         print(f"⬇️  Downloading video...")
         print(f"Original URI: {video_uri}")
-        
+
         # Convert Google URI to LiteLLM proxy URI
         # Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media
         if video_uri.startswith("files/"):
             download_path = f"{video_uri}:download?alt=media"
         else:
             download_path = video_uri
-            
+
         litellm_download_url = f"{self.base_url}/{download_path}"
         print(f"Download URL: {litellm_download_url}")
-        
+
         try:
             # Download with streaming and redirect handling
             response = requests.get(
-                litellm_download_url, 
-                headers=self.headers, 
+                litellm_download_url,
+                headers=self.headers,
                 stream=True,
-                allow_redirects=True  # Handle redirects automatically
+                allow_redirects=True,  # Handle redirects automatically
             )
             response.raise_for_status()
-            
+
             # Save video file
-            with open(output_filename, 'wb') as f:
+            with open(output_filename, "wb") as f:
                 downloaded_size = 0
                 for chunk in response.iter_content(chunk_size=8192):
                     if chunk:
                         f.write(chunk)
                         downloaded_size += len(chunk)
-                        
+
                         # Progress indicator for large files
                         if downloaded_size % (1024 * 1024) == 0:  # Every MB
-                            print(f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB...")
-            
+                            print(
+                                f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB..."
+                            )
+
             # Verify file was created and has content
             if os.path.exists(output_filename):
                 file_size = os.path.getsize(output_filename)
@@ -203,48 +209,52 @@ class VeoVideoGenerator:
             else:
                 print("❌ File was not created")
                 return False
-                
+
         except requests.RequestException as e:
             print(f"❌ Download failed: {e}")
-            if hasattr(e, 'response') and e.response is not None:
+            if hasattr(e, "response") and e.response is not None:
                 print(f"Status code: {e.response.status_code}")
                 print(f"Response headers: {dict(e.response.headers)}")
             return False
-    
+
     def generate_and_download(self, prompt: str, output_filename: str = None) -> bool:
         """
         Complete workflow: generate video and download it.
-        
+
         Args:
             prompt: Text description for video generation
             output_filename: Output filename (auto-generated if None)
-            
+
         Returns:
             True if successful, False otherwise
         """
         # Auto-generate filename if not provided
         if output_filename is None:
             timestamp = int(time.time())
-            safe_prompt = "".join(c for c in prompt[:30] if c.isalnum() or c in (' ', '-', '_')).rstrip()
-            output_filename = f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
-        
+            safe_prompt = "".join(
+                c for c in prompt[:30] if c.isalnum() or c in (" ", "-", "_")
+            ).rstrip()
+            output_filename = (
+                f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
+            )
+
         print("=" * 60)
         print("🎬 VEO VIDEO GENERATION WORKFLOW")
         print("=" * 60)
-        
+
         # Step 1: Generate video
         operation_name = self.generate_video(prompt)
         if not operation_name:
             return False
-        
+
         # Step 2: Wait for completion
         video_uri = self.wait_for_completion(operation_name)
         if not video_uri:
             return False
-        
+
         # Step 3: Download video
         success = self.download_video(video_uri, output_filename)
-        
+
         if success:
             print("=" * 60)
             print("🎉 SUCCESS! Video generation complete!")
@@ -254,51 +264,51 @@ class VeoVideoGenerator:
             print("=" * 60)
             print("❌ FAILED! Video generation or download failed")
             print("=" * 60)
-        
+
         return success
 
 
 def main():
     """
     Example usage of the VeoVideoGenerator.
-    
+
     Configure these environment variables:
     - LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
     - LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
     """
-    
+
     # Configuration from environment or defaults
     base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
     api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
-    
+
     print("🚀 Starting Veo Video Generation Example")
     print(f"📡 Using LiteLLM proxy at: {base_url}")
-    
+
     # Initialize generator
     generator = VeoVideoGenerator(base_url=base_url, api_key=api_key)
-    
+
     # Example prompts - try different ones!
     example_prompts = [
         "A cat playing with a ball of yarn in a sunny garden",
         "Ocean waves crashing against rocky cliffs at sunset",
         "A bustling city street with people walking and cars passing by",
-        "A peaceful forest with sunlight filtering through the trees"
+        "A peaceful forest with sunlight filtering through the trees",
     ]
-    
+
     # Use first example or get from user
     prompt = example_prompts[0]
     print(f"🎬 Using prompt: '{prompt}'")
-    
+
     # Generate and download video
     success = generator.generate_and_download(prompt)
-    
+
     if success:
         print("\n✅ Example completed successfully!")
         print("💡 Try modifying the prompt in the script for different videos!")
     else:
         print("\n❌ Example failed!")
         print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key")
-        
+
         # Troubleshooting tips
         print("\n🔍 Troubleshooting:")
         print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through")

deploy/Dockerfile.ghcr_base

@@ -1,18 +0,0 @@
-# Use the provided base image
-FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186
-
-# Set the working directory to /app
-WORKDIR /app
-
-# Copy the configuration file into the container at /app
-COPY config.yaml .
-
-# Make sure your docker/entrypoint.sh is executable
-# Convert Windows line endings to Unix
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
-
-# Expose the necessary port
-EXPOSE 4000/tcp
-
-# Override the CMD instruction with your desired command and arguments
-CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug", "--run_gunicorn"]

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -47,7 +47,7 @@ spec:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       {{- with .Values.extraInitContainers }}
       initContainers:
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       containers:
         - name: {{ include "litellm.name" . }}
@@ -100,6 +100,16 @@ spec:
             - name: DATABASE_URL
               value: {{ .Values.db.url | quote }}
             {{- end }}
+            {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
+            - name: DATABASE_URL_READ_REPLICA
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.secret.name }}
+                  key: {{ .Values.db.secret.readReplicaUrlKey }}
+            {{- else if .Values.db.readReplicaUrl }}
+            - name: DATABASE_URL_READ_REPLICA
+              value: {{ .Values.db.readReplicaUrl | quote }}
+            {{- end }}
             - name: PROXY_MASTER_KEY
               valueFrom:
                 secretKeyRef:
@@ -116,21 +126,32 @@ spec:
                   name: {{ include "redis.secretName" .Subcharts.redis }}
                   key: {{include "redis.secretPasswordKey" .Subcharts.redis }}
             {{- end }}
+            {{- /*
+              Inject LITELLM_LOG only when envVars does not already define it.
+            */}}
+            {{- if and .Values.logLevel (not (hasKey (default dict .Values.envVars) "LITELLM_LOG")) }}
+            - name: LITELLM_LOG
+              value: {{ .Values.logLevel | quote }}
+            {{- end }}
             {{- if .Values.envVars }}
             {{- range $key, $val := .Values.envVars }}
             - name: {{ $key }}
               value: {{ $val | quote }}
             {{- end }}
             {{- end }}
-            {{- if .Values.separateHealthApp }}
-            - name: SEPARATE_HEALTH_APP
-              value: "1"
-            - name: SEPARATE_HEALTH_PORT
-              value: {{ .Values.separateHealthPort | default "8081" | quote }}
-            {{- end }}
             {{- with .Values.extraEnvVars }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
+            {{- if .Values.migrationJob.enabled }}
+            # Schema updates are owned by the dedicated migrations Job; skip
+            # the proxy's startup `prisma db push` so N replicas don't race
+            # one DB on every rollout. Placed last (after envVars and
+            # extraEnvVars) so this override can't be silently shadowed by a
+            # user-supplied DISABLE_SCHEMA_UPDATE under last-wins duplicate-env
+            # semantics — same pattern the migrations Job uses.
+            - name: DISABLE_SCHEMA_UPDATE
+              value: "true"
+            {{- end }}
           envFrom:
           {{- range .Values.environmentSecrets }}
             - secretRef:
@@ -158,15 +179,10 @@ spec:
             - name: http
               containerPort: {{ .Values.service.port }}
               protocol: TCP
-            {{- if .Values.separateHealthApp }}
-            - name: health
-              containerPort: {{ .Values.separateHealthPort | default 8081 }}
-              protocol: TCP
-            {{- end }}
           livenessProbe:
             httpGet:
               path: {{ .Values.livenessProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
@@ -175,7 +191,7 @@ spec:
           readinessProbe:
             httpGet:
               path: {{ .Values.readinessProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
@@ -184,7 +200,7 @@ spec:
           startupProbe:
             httpGet:
               path: {{ .Values.startupProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.startupProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
@@ -212,7 +228,7 @@ spec:
             {{- toYaml . | nindent 12 }}
           {{- end }}
       {{- with .Values.extraContainers }}
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       volumes:
         {{ if .Values.securityContext.readOnlyRootFilesystem }}

deploy/charts/litellm-helm/templates/hpa.yaml

@@ -12,6 +12,10 @@ spec:
     name: {{ include "litellm.fullname" . }}
   minReplicas: {{ .Values.autoscaling.minReplicas }}
   maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+  {{- if .Values.autoscaling.behavior }}
+  behavior:
+    {{- toYaml .Values.autoscaling.behavior | nindent 4 }}
+  {{- end }}
   metrics:
     {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
     - type: Resource

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -37,7 +37,7 @@ spec:
       serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
       {{- with .Values.migrationJob.extraInitContainers }}
       initContainers:
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }} 
       containers:
         - name: prisma-migrations
@@ -96,7 +96,7 @@ spec:
             {{- toYaml . | nindent 12 }}
           {{- end }}
       {{- with .Values.migrationJob.extraContainers }}
-        {{- toYaml . | nindent 8 }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
       {{- with .Values.volumes }}
       volumes:

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -257,16 +257,16 @@ tests:
           value: 0
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.periodSeconds
-          value: 10
+          value: 15
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
-          value: 1
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.successThreshold
           value: 1
       - equal:
           path: spec.template.spec.containers[0].livenessProbe.failureThreshold
-          value: 3
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].readinessProbe.httpGet.path
           value: /health/readiness
@@ -278,7 +278,7 @@ tests:
           value: 10
       - equal:
           path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
-          value: 1
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].readinessProbe.successThreshold
           value: 1
@@ -296,7 +296,7 @@ tests:
           value: 10
       - equal:
           path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
-          value: 1
+          value: 5
       - equal:
           path: spec.template.spec.containers[0].startupProbe.successThreshold
           value: 1
@@ -319,3 +319,61 @@ tests:
     asserts:
       - notExists:
           path: spec.minReadySeconds
+  - it: should work with extraInitContainers
+    template: deployment.yaml
+    set:
+      extraInitContainers:
+        - name: init-test
+          image: busybox:latest
+          command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-test
+            image: busybox:latest
+            command: ["echo", "hello"]
+  - it: should support tpl in extraInitContainers
+    template: deployment.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      extraInitContainers:
+        - name: init-tpl
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-tpl
+            image: "ghcr.io/berriai/litellm-database:test"
+            command: ["echo", "hello"]
+  - it: should work with extraContainers
+    template: deployment.yaml
+    set:
+      extraContainers:
+        - name: sidecar
+          image: busybox:latest
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar
+            image: busybox:latest
+  - it: should support tpl in extraContainers
+    template: deployment.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      extraContainers:
+        - name: sidecar-tpl
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar-tpl
+            image: "ghcr.io/berriai/litellm-database:test"

deploy/charts/litellm-helm/tests/hpa_tests.yaml

@@ -0,0 +1,36 @@
+suite: "hpa with behavior"
+templates:
+  - hpa.yaml
+tests:
+  - it: "renders behavior when set"
+    set:
+      autoscaling.enabled: true
+      autoscaling.behavior:
+        scaleUp:
+          stabilizationWindowSeconds: 60
+          policies:
+            - type: Pods
+              value: 2
+              periodSeconds: 60
+        scaleDown:
+          stabilizationWindowSeconds: 90
+          policies:
+            - type: Pods
+              value: 1
+              periodSeconds: 60
+    asserts:
+      - isKind: { of: HorizontalPodAutoscaler }
+      - equal: { path: spec.behavior.scaleUp.stabilizationWindowSeconds, value: 60 }
+      - equal: { path: spec.behavior.scaleDown.stabilizationWindowSeconds, value: 90 }
+
+---
+suite: "hpa without behavior"
+templates:
+  - hpa.yaml
+tests:
+  - it: "does not render behavior when not set"
+    set:
+      autoscaling.enabled: true
+    asserts:
+      - isKind: { of: HorizontalPodAutoscaler }
+      - isNull: { path: spec.behavior }

deploy/charts/litellm-helm/tests/migrations-job_tests.yaml

@@ -188,3 +188,69 @@ tests:
       - equal:
           path: spec.template.spec.serviceAccountName
           value: pre-existing-sa
+  - it: should work with extraInitContainers
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        extraInitContainers:
+          - name: init-test
+            image: busybox:latest
+            command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-test
+            image: busybox:latest
+            command: ["echo", "hello"]
+  - it: should support tpl in extraInitContainers
+    template: migrations-job.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      migrationJob:
+        enabled: true
+        extraInitContainers:
+          - name: init-tpl
+            image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+            command: ["echo", "hello"]
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers
+          content:
+            name: init-tpl
+            image: "ghcr.io/berriai/litellm-database:test"
+            command: ["echo", "hello"]
+  - it: should work with extraContainers
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        extraContainers:
+          - name: sidecar
+            image: busybox:latest
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar
+            image: busybox:latest
+  - it: should support tpl in extraContainers
+    template: migrations-job.yaml
+    set:
+      image:
+        repository: ghcr.io/berriai/litellm-database
+        tag: test
+      migrationJob:
+        enabled: true
+        extraContainers:
+          - name: sidecar-tpl
+            image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+    asserts:
+      - contains:
+          path: spec.template.spec.containers
+          content:
+            name: sidecar-tpl
+            image: "ghcr.io/berriai/litellm-database:test"

deploy/charts/litellm-helm/values.yaml

@@ -88,26 +88,20 @@ service:
   # optionally specify loadBalancerClass
   # loadBalancerClass: tailscale
 
-# Separate health app configuration
-# When enabled, health checks will use a separate port and the application
-# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
-separateHealthApp: false
-separateHealthPort: 8081
-
-# Probe tuning for proxy container
+# Probes for LiteLLM gateway container
 livenessProbe:
   path: /health/liveliness
   initialDelaySeconds: 0
-  periodSeconds: 10
-  timeoutSeconds: 1
+  periodSeconds: 15
+  timeoutSeconds: 5
   successThreshold: 1
-  failureThreshold: 3
+  failureThreshold: 5
 
 readinessProbe:
   path: /health/readiness
   initialDelaySeconds: 0
   periodSeconds: 10
-  timeoutSeconds: 1
+  timeoutSeconds: 5
   successThreshold: 1
   failureThreshold: 3
 
@@ -115,7 +109,7 @@ startupProbe:
   path: /health/readiness
   initialDelaySeconds: 0
   periodSeconds: 10
-  timeoutSeconds: 1
+  timeoutSeconds: 5
   successThreshold: 1
   failureThreshold: 30
 
@@ -190,6 +184,7 @@ autoscaling:
   maxReplicas: 100
   targetCPUUtilizationPercentage: 80
   # targetMemoryUtilizationPercentage: 80
+  # behavior: {}
 
 # Autoscaling with keda is mutually exclusive with hpa
 keda:
@@ -258,6 +253,26 @@ db:
     passwordKey: password
     # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
     endpointKey: ""
+    # Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this
+    # secret key instead of db.readReplicaUrl. Prefer this over the plain
+    # value: read-replica URLs typically embed credentials, and a value
+    # written to db.readReplicaUrl ends up visible in the rendered pod spec
+    # and the Helm release secret.
+    readReplicaUrlKey: ""
+
+  # Optional read-replica routing. When set, the proxy sends read-only
+  # queries (find_*, count, group_by, query_raw/_first) to this URL while
+  # writes continue to go to db.url. Useful for Aurora-style clusters with
+  # separate reader/writer endpoints. Leave empty to keep single-DB behavior.
+  # When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed
+  # alongside the writer (host/port/user/db are parsed from this URL once
+  # at startup; only the IAM token rotates).
+  #
+  # If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over
+  # this field — the plain value is rendered into the pod spec and the
+  # Helm release secret. This field is intended for credential-less URLs
+  # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime).
+  readReplicaUrl: ""
 
   # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
   #  The Stackgres Operator must already be installed within the target
@@ -329,6 +344,17 @@ migrationJob:
     helm:
       enabled: false
 
+# Log level for the litellm proxy (sets LITELLM_LOG in the deployment env).
+# Rendered as a direct `env:` entry, which in Kubernetes takes precedence over
+# any `envFrom:` source. If you currently source LITELLM_LOG from an
+# environmentSecret or environmentConfigMap, set `logLevel: ""` here to
+# disable injection — otherwise this value silently overrides your secret /
+# configmap entry.
+#
+# Setting LITELLM_LOG inside `envVars:` below also wins: the template skips
+# this injection entirely when envVars already defines LITELLM_LOG.
+logLevel: INFO
+
 # Additional environment variables to be added to the deployment as a map of key-value pairs
 envVars: {}
 

deploy/kubernetes/kub.yaml

@@ -1,56 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: litellm-deployment
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: litellm
-  template:
-    metadata:
-      labels:
-        app: litellm
-    spec:
-      containers:
-        - name: litellm-container
-          image: ghcr.io/berriai/litellm:main-latest
-          imagePullPolicy: Always
-          env:
-            - name: AZURE_API_KEY
-              value: "d6f****"
-            - name: AZURE_API_BASE
-              value: "https://openai"
-            - name: LITELLM_MASTER_KEY
-              value: "sk-1234"
-            - name: DATABASE_URL
-              value: "postgresql://ishaan*********"
-          args:
-            - "--config"
-            - "/app/proxy_config.yaml"  # Update the path to mount the config file
-          volumeMounts:                 # Define volume mount for proxy_config.yaml
-            - name: config-volume
-              mountPath: /app
-              readOnly: true
-          livenessProbe:
-            httpGet:
-              path: /health/liveliness
-              port: 4000
-            initialDelaySeconds: 120
-            periodSeconds: 15
-            successThreshold: 1
-            failureThreshold: 3
-            timeoutSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /health/readiness
-              port: 4000
-            initialDelaySeconds: 120
-            periodSeconds: 15
-            successThreshold: 1
-            failureThreshold: 3
-            timeoutSeconds: 10
-      volumes:  # Define volume to mount proxy_config.yaml
-        - name: config-volume
-          configMap:
-            name: litellm-config  

deploy/kubernetes/service.yaml

@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: litellm-service
-spec:
-  selector:
-    app: litellm
-  ports:
-    - protocol: TCP
-      port: 4000
-      targetPort: 4000
-  type: LoadBalancer

dev_config.yaml

@@ -1,13 +0,0 @@
-model_list:
-  - model_name: fake-openai-endpoint
-    litellm_params:
-      model: openai/fake-model
-      api_key: fake-key
-      api_base: https://exampleopenaiendpoint-production.up.railway.app/
-
-general_settings:
-  master_key: sk-1234
-
-litellm_settings:
-  drop_params: True
-  telemetry: False

docker-compose.yml

@@ -16,6 +16,11 @@ services:
       - "4000:4000" # Map the container port to the host, change the host port if necessary
     environment:
       DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
+      # Optional: route read-only queries (find_*, count, group_by, query_raw/_first)
+      # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for
+      # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL
+      # is auto-refreshed alongside the writer.
+      # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm"
       STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
     env_file:
       - .env # Load local .env file

docker/Dockerfile.alpine

@@ -1,57 +0,0 @@
-# Base image for building
-ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
-
-# Runtime image
-ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
-
-# Builder stage
-FROM $LITELLM_BUILD_IMAGE AS builder
-
-# Set the working directory to /app
-WORKDIR /app
-
-# Install build dependencies
-RUN apk add --no-cache gcc python3-dev musl-dev
-
-RUN pip install --upgrade pip==26.0.1 && \
-    pip install build==1.4.2
-
-# Copy the current directory contents into the container at /app
-COPY . .
-
-# Build the package
-RUN rm -rf dist/* && python -m build
-
-# There should be only one wheel file now, assume the build only creates one
-RUN ls -1 dist/*.whl | head -1
-
-# Install the package
-RUN pip install dist/*.whl
-
-# install dependencies as wheels
-RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
-
-# Runtime stage
-FROM $LITELLM_RUNTIME_IMAGE AS runtime
-
-# Update dependencies and clean up, install libsndfile for audio processing
-RUN apk upgrade --no-cache && apk add --no-cache libsndfile
-
-WORKDIR /app
-
-# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
-COPY --from=builder /app/dist/*.whl .
-COPY --from=builder /wheels/ /wheels/
-
-# Install the built wheel using pip; again using a wildcard if it's the only file
-RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels
-
-# Convert Windows line endings to Unix for entrypoint scripts
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
-RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
-
-EXPOSE 4000/tcp
-
-# Set your entrypoint and command
-ENTRYPOINT ["docker/prod_entrypoint.sh"]
-CMD ["--port", "4000"]

docker/Dockerfile.custom_ui

@@ -1,78 +0,0 @@
-# Use the provided base image
-# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt.
-FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
-
-# Set the working directory to /app
-WORKDIR /app
-
-# Install Node.js and npm (adjust version as needed)
-RUN apt-get update && apt-get upgrade -y \
-        libxml2 \
-        libexpat1 \
-        openssl \
-        libssl3 \
-        git \
-        libkrb5-3 \
-        libglib2.0-0 \
-        wget \
-        libaom3 \
-        libxslt1.1 \
-        libgnutls30 \
-        libc6 && \
-    apt-get install -y --no-install-recommends nodejs npm && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
-    GLOBAL="$(npm root -g)" && \
-    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done && \
-    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
-    npm cache clean --force && \
-    apt-get purge -y npm
-
-# Copy the UI source into the container
-COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard
-
-# Set an environment variable for UI_BASE_PATH
-# This can be overridden at build time
-# set UI_BASE_PATH to "<your server root path>/ui"
-ENV UI_BASE_PATH="/prod/ui"
-
-# Build the UI with the specified UI_BASE_PATH
-WORKDIR /app/ui/litellm-dashboard
-RUN npm ci
-RUN UI_BASE_PATH=$UI_BASE_PATH npm run build
-
-# Create the destination directory
-RUN mkdir -p /app/litellm/proxy/_experimental/out
-
-# Move the built files to the appropriate location
-# Assuming the build output is in ./out directory
-RUN rm -rf /app/litellm/proxy/_experimental/out/* && \
-    mv ./out/* /app/litellm/proxy/_experimental/out/
-
-# Switch back to the main app directory
-WORKDIR /app
-
-# Make sure your docker/entrypoint.sh is executable
-# Convert Windows line endings to Unix for entrypoint scripts
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
-RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
-
-# Expose the necessary port
-EXPOSE 4000/tcp
-
-# Override the CMD instruction with your desired command and arguments
-CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]

docker/Dockerfile.database

@@ -1,55 +1,72 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
-# Builder stage
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
+
+FROM $UV_IMAGE AS uvbin
+
 FROM $LITELLM_BUILD_IMAGE AS builder
 
-# Set the working directory to /app
 WORKDIR /app
-
 USER root
 
-# Install build dependencies
+COPY --from=uvbin /uv /usr/local/bin/uv
+COPY --from=uvbin /uvx /usr/local/bin/uvx
+
 RUN apk add --no-cache \
     bash \
     gcc \
-    py3-pip \
     python3 \
     python3-dev \
     openssl \
-    openssl-dev
+    openssl-dev \
+    nodejs \
+    npm \
+    libsndfile
 
-RUN python -m pip install build==1.4.2
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
+    UV_LINK_MODE=copy \
+    PATH="/app/.venv/bin:${PATH}"
 
-# Copy the current directory contents into the container at /app
+# Copy dependency metadata first for layer caching
+COPY pyproject.toml uv.lock ./
+COPY enterprise/pyproject.toml enterprise/
+COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
+
+# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
+RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
+    --extra proxy \
+    --extra proxy-runtime \
+    --extra extra_proxy \
+    --extra semantic-router \
+    --python python3
+
+# Copy full source tree
 COPY . .
 
-# Build Admin UI
-# Convert Windows line endings to Unix and make executable
+# Build Admin UI before final sync
 RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
-# Build the package
-RUN rm -rf dist/* && python -m build
+# Install project and workspace packages (fast - deps already cached)
+RUN uv sync --frozen --no-default-groups --no-editable \
+    --extra proxy \
+    --extra proxy-runtime \
+    --extra extra_proxy \
+    --extra semantic-router \
+    --python python3
 
-# There should be only one wheel file now, assume the build only creates one
-RUN ls -1 dist/*.whl | head -1
+RUN prisma generate --schema=./schema.prisma
 
-# Install the package
-RUN pip install dist/*.whl
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
+    sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
-# install dependencies as wheels
-RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
-
-# Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
-# Ensure runtime stage runs as root
 USER root
 
-# Install runtime dependencies
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
     npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
@@ -67,72 +84,23 @@ RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile
     find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
         rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
     done && \
-    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
     npm cache clean --force && \
     { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
-# Copy the current directory contents into the container at /app
-COPY . .
-RUN ls -la /app
+ENV PATH="/app/.venv/bin:${PATH}"
 
-# Copy the built wheel from the builder stage to the runtime stage; assumes only one wheel file is present
-COPY --from=builder /app/dist/*.whl .
-COPY --from=builder /wheels/ /wheels/
+COPY --from=builder /app /app
+# Prisma binaries live in $HOME/.cache (default prisma-python location),
+# which is /root/.cache here. Copy them from the builder so they survive
+# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
+# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
+COPY --from=builder /root/.cache /root/.cache
 
-# Install the built wheel using pip; again using a wildcard if it's the only file
-RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && rm -f *.whl && rm -rf /wheels
+RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
+    find /app/.venv -type d -path "*/tornado/test" -delete
 
-# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
-# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
-# Patch every copy of tar, glob, and brace-expansion inside that tree.
-RUN GLOBAL="$(npm root -g)" && \
-    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
-    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-    done && \
-    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done
-
-# Install semantic_router and aurelio-sdk using script
-# Convert Windows line endings to Unix and make executable
-RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
-
-# ensure pyjwt is used, not jwt
-RUN pip uninstall jwt -y
-RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.12.0 --no-cache-dir
-
-# Build Admin UI (runtime stage)
-# Convert Windows line endings to Unix and make executable
-RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-
-# Generate prisma client
-RUN prisma generate
-# Convert Windows line endings to Unix for entrypoint scripts
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
-RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 EXPOSE 4000/tcp
 
-RUN apk add --no-cache supervisor
-COPY docker/supervisord.conf /etc/supervisord.conf
-
-# # Set your entrypoint and command
-
-
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
-
-# Append "--detailed_debug" to the end of CMD to view detailed debug logs 
-# CMD ["--port", "4000", "--detailed_debug"]
 CMD ["--port", "4000"]

docker/Dockerfile.dev

@@ -1,151 +0,0 @@
-# Base image for building
-ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-
-# Runtime image
-ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-
-# Builder stage
-FROM $LITELLM_BUILD_IMAGE AS builder
-
-# Set the working directory to /app
-WORKDIR /app
-
-USER root
-
-# Install build dependencies in one layer
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gcc \
-    g++ \
-    python3-dev \
-    libssl-dev \
-    pkg-config \
-    && rm -rf /var/lib/apt/lists/* \
-    && pip install --upgrade pip==26.0.1 build==1.4.2
-
-# Copy requirements first for better layer caching
-COPY requirements.txt .
-
-# Install Python dependencies with cache mount for faster rebuilds
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
-
-# Fix JWT dependency conflicts early
-RUN pip uninstall jwt -y || true && \
-    pip uninstall PyJWT -y || true && \
-    pip install PyJWT==2.12.0 --no-cache-dir
-
-# Copy only necessary files for build
-COPY pyproject.toml README.md schema.prisma poetry.lock ./
-COPY litellm/ ./litellm/
-COPY enterprise/ ./enterprise/
-COPY docker/ ./docker/
-
-# Build Admin UI once
-# Convert Windows line endings to Unix and make executable
-RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-
-# Build the package
-RUN rm -rf dist/* && python -m build
-
-# Install the built package
-RUN pip install dist/*.whl
-
-# Runtime stage
-FROM $LITELLM_RUNTIME_IMAGE AS runtime
-
-# Ensure runtime stage runs as root
-USER root
-
-# Install only runtime dependencies
-RUN apt-get update && apt-get upgrade -y \
-        libxml2 \
-        libexpat1 \
-        openssl \
-        libssl3 \
-        git \
-        libkrb5-3 \
-        libglib2.0-0 \
-        wget \
-        libaom3 \
-        libxslt1.1 \
-        libgnutls30 \
-        libc6 \
-    && apt-get install -y --no-install-recommends \
-        libssl3 \
-    libatomic1 \
-    nodejs \
-    npm \
-    && rm -rf /var/lib/apt/lists/* \
-    && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
-    && GLOBAL="$(npm root -g)" \
-    && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-       done \
-    && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
-    && npm cache clean --force \
-    && apt-get purge -y npm
-
-WORKDIR /app
-
-# Copy only necessary runtime files
-COPY docker/entrypoint.sh docker/prod_entrypoint.sh ./docker/
-COPY litellm/ ./litellm/
-COPY pyproject.toml README.md schema.prisma poetry.lock ./
-
-# Copy pre-built wheels and install everything at once
-COPY --from=builder /wheels/ /wheels/
-COPY --from=builder /app/dist/*.whl .
-
-# Install all dependencies in one step with no-cache for smaller image
-RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/ --no-deps && \
-    rm -f *.whl && \
-    rm -rf /wheels
-
-# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
-# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
-# Patch every copy of tar, glob, and brace-expansion inside that tree.
-RUN GLOBAL="$(npm root -g)" && \
-    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
-    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-    done && \
-    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done
-
-# Generate prisma client and set permissions
-# Convert Windows line endings to Unix for entrypoint scripts
-RUN prisma generate && \
-    sed -i 's/\r$//' docker/entrypoint.sh && \
-    sed -i 's/\r$//' docker/prod_entrypoint.sh && \
-    chmod +x docker/entrypoint.sh && \
-    chmod +x docker/prod_entrypoint.sh
-
-EXPOSE 4000/tcp
-
-ENTRYPOINT ["docker/prod_entrypoint.sh"]
-
-# Append "--detailed_debug" to the end of CMD to view detailed debug logs 
-CMD ["--port", "4000"]

docker/Dockerfile.health_check

@@ -1,24 +0,0 @@
-FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-
-WORKDIR /app
-
-# Copy health check script and requirements
-COPY scripts/health_check/health_check_client.py /app/health_check_client.py
-COPY scripts/health_check/health_check_requirements.txt /app/requirements.txt
-
-# Install dependencies
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Make script executable
-RUN chmod +x /app/health_check_client.py
-
-# Run as non-root user
-RUN adduser --disabled-password --gecos "" --uid 1001 healthcheck
-USER healthcheck
-
-# Health check
-HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
-  CMD python /app/health_check_client.py --help || exit 1
-
-# Set entrypoint
-ENTRYPOINT ["python", "/app/health_check_client.py"]

docker/Dockerfile.non_root

@@ -1,67 +1,65 @@
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a5a619c1793039dcf92f02178f37c94bb3d6001403716da59d6092dfe8d9b502
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
 ARG PROXY_EXTRAS_SOURCE=published
+ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
+
+FROM $UV_IMAGE AS uvbin
 
-# -----------------
-# Builder Stage
-# -----------------
 FROM $LITELLM_BUILD_IMAGE AS builder
 ARG PROXY_EXTRAS_SOURCE
 WORKDIR /app
 USER root
 
-# Install build dependencies with retry logic (includes node for UI build)
+COPY --from=uvbin /uv /usr/local/bin/uv
+COPY --from=uvbin /uvx /usr/local/bin/uvx
+
 RUN for i in 1 2 3; do \
-    apk add --no-cache \
-    python3 \
-    python3-dev \
-    py3-pip \
-    clang \
-    llvm \
-    lld \
-    gcc \
-    linux-headers \
-    build-base \
-    bash \
-    nodejs \
-    npm && break || sleep 5; \
-    done \
-  && pip install --no-cache-dir --upgrade pip==26.0.1 build==1.4.2
+      apk add --no-cache \
+        python3 \
+        python3-dev \
+        gcc \
+        bash \
+        coreutils \
+        curl \
+        openssl \
+        libsndfile \
+        nodejs && break || sleep 5; \
+    done
 
-# Cache Python dependencies
-COPY requirements.txt .
-RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \
-  && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.12.0"
+ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
+    UV_LINK_MODE=copy \
+    PATH="/app/.venv/bin:${PATH}" \
+    LITELLM_NON_ROOT=true \
+    PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
+    XDG_CACHE_HOME=/app/.cache
 
-# Copy source after dependency layers
+# Copy dependency metadata first for layer caching
+COPY pyproject.toml uv.lock ./
+COPY enterprise/pyproject.toml enterprise/
+COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
+
+# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
+RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
+    uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
+    --extra proxy \
+    --extra proxy-runtime \
+    --extra extra_proxy \
+    --extra semantic-router \
+    --python python3
+
+# Copy full source tree
 COPY . .
 
 # Set non-root flag for build time consistency
 ENV LITELLM_NON_ROOT=true
 
-# Build Admin UI using the upstream command order while keeping a single RUN layer
-# NOTE: .npmrc files (which may set ignore-scripts=true and min-release-age=3d)
-# are temporarily renamed during npm install/ci so they don't block lifecycle
-# scripts needed by the build. This is safe because npm ci installs from
-# package-lock.json with pinned versions + integrity hashes.
-RUN mkdir -p /var/lib/litellm/ui && \
-    ([ -f /app/.npmrc ] && mv /app/.npmrc /app/.npmrc.bak || true) && \
-    npm install -g npm@11.12.1 && \
-    npm install -g node-gyp@12.2.0 && \
-    ln -sf "$(npm root -g)/node-gyp" "$(npm root -g)/npm/node_modules/node-gyp" && \
-    npm cache clean --force && \
-    cd /app/ui/litellm-dashboard && \
-    if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
-      cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
-    fi && \
-    ([ -f .npmrc ] && mv .npmrc .npmrc.bak || true) && \
-    npm ci && \
-    ([ -f .npmrc.bak ] && mv .npmrc.bak .npmrc || true) && \
-    ([ -f /app/.npmrc.bak ] && mv /app/.npmrc.bak /app/.npmrc || true) && \
-    npm run build && \
-    cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
-    mkdir -p /var/lib/litellm/assets && \
+# Stage the pre-built Admin UI from the checked-in Next.js static export.
+# _experimental/out/ is regenerated as part of the release runbook.
+# Restructure extensionless routes (foo.html -> foo/index.html) to match the layout
+# proxy_server.py expects, and drop a readiness marker.
+RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
+    cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
     cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
     ( cd /var/lib/litellm/ui && \
       for html_file in *.html; do \
@@ -71,178 +69,77 @@ RUN mkdir -p /var/lib/litellm/ui && \
           mv "$html_file" "$folder_name/index.html"; \
         fi; \
       done && \
-      touch .litellm_ui_ready ) && \
-    cd /app/ui/litellm-dashboard && rm -rf ./out
+      touch .litellm_ui_ready )
 
-# Build litellm wheel and place it in wheels dir (replace any PyPI wheels)
-RUN rm -rf dist/* && python -m build && \
-  rm -f /wheels/litellm-*.whl && \
-  cp dist/*.whl /wheels/
-
-# Optionally build local litellm-proxy-extras wheel
-RUN if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \
-      cd /app/litellm-proxy-extras && rm -rf dist && python -m build && \
-      cp dist/*.whl /wheels/; \
+RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
+    if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
+      uv sync --frozen --no-default-groups --no-editable \
+        --extra proxy \
+        --extra proxy-runtime \
+        --extra extra_proxy \
+        --extra semantic-router \
+        --python python3 \
+        --no-sources-package litellm-proxy-extras; \
+    else \
+      uv sync --frozen --no-default-groups --no-editable \
+        --extra proxy \
+        --extra proxy-runtime \
+        --extra extra_proxy \
+        --extra semantic-router \
+        --python python3; \
     fi
 
-# Pre-cache Prisma binaries in the builder stage
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
+RUN prisma generate --schema=./schema.prisma
 
-RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.13.1 \
-    && mkdir -p /app/.cache/npm
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
+    sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
-RUN NPM_CONFIG_CACHE=/app/.cache/npm \
-    python -c "import prisma.cli.prisma as p; p.ensure_cached()"
-
-RUN prisma generate && \
-    prisma --version && \
-    prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true
-
-# -----------------
-# Runtime Stage
-# -----------------
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 ARG PROXY_EXTRAS_SOURCE
 WORKDIR /app
 USER root
 
-# Install runtime dependencies with retry
 RUN for i in 1 2 3; do \
-    apk upgrade --no-cache && break || sleep 5; \
-    done \
-  && for i in 1 2 3; do \
-    apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
-    done \
-  && apk upgrade --no-cache nodejs \
-  && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
-  && GLOBAL="$(npm root -g)" \
-  && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-     done \
-  && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-     done \
-  && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-     done \
-  && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-     done \
-  && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-     done \
-  && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
-  && npm cache clean --force \
-  && { apk del --no-cache npm 2>/dev/null || true; }
-
-# Copy artifacts from builder
-COPY --from=builder /app/requirements.txt /app/requirements.txt
-COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
-COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
-COPY --from=builder /app/schema.prisma /app/
-# Keep enterprise bridge module in runtime so `enterprise.enterprise_hooks`
-# can load and register managed enterprise hooks (e.g. managed_files).
-COPY --from=builder /app/enterprise /app/enterprise
-# Copy prisma_migration.py for Helm migrations job compatibility
-COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
-COPY --from=builder /wheels/ /wheels/
-COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
-COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
-COPY --from=builder /app/.cache /app/.cache
-COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
-COPY --from=builder \
-  /usr/lib/python3.13/site-packages/nodejs* \
-  /usr/lib/python3.13/site-packages/prisma* \
-  /usr/lib/python3.13/site-packages/tomlkit* \
-  /usr/lib/python3.13/site-packages/nodeenv* \
-  /usr/lib/python3.13/site-packages/
-COPY --from=builder /usr/bin/prisma /usr/bin/prisma
-
-# Final runtime environment configuration
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
-    HOME=/app \
-    LITELLM_NON_ROOT=true \
-    XDG_CACHE_HOME=/app/.cache
-
-# Install packages from wheels and optional extras without network
-RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
-    pip install --no-index --find-links=/wheels/ /wheels/litellm-*-py3-none-any.whl && \
-    pip install --no-index --find-links=/wheels/ --no-deps semantic_router==0.1.11 && \
-    pip install --no-index --find-links=/wheels/ aurelio-sdk==0.0.19 && \
-    if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \
-      if ls /wheels/litellm_proxy_extras-*.whl >/dev/null 2>&1; then \
-        pip install --no-index --find-links=/wheels/ /wheels/litellm_proxy_extras-*.whl; \
-      else \
-        echo "litellm_proxy_extras wheel not found; skipping local install"; \
-      fi; \
-    fi
-
-# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
-# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
-# Patch every copy of tar, glob, and brace-expansion inside that tree.
-RUN GLOBAL="$(npm root -g)" && \
-    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
-    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+      apk upgrade --no-cache && break || sleep 5; \
     done && \
-    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    for i in 1 2 3; do \
+      apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
     done
 
-# Permissions, cleanup, and Prisma prep
-# Convert Windows line endings to Unix for entrypoint scripts
-RUN sed -i 's/\r$//' docker/entrypoint.sh && \
-    sed -i 's/\r$//' docker/prod_entrypoint.sh && \
-    chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
-    mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \
-    chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \
-    pip uninstall jwt -y || true && \
-    pip uninstall PyJWT -y || true && \
-    pip install --no-index --find-links=/wheels/ PyJWT==2.12.0 --no-cache-dir && \
-    rm -rf /wheels && \
-    PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
-    chown -R nobody:nogroup $PRISMA_PATH && \
-    LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
-    [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \
-    LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
-    chgrp -R 0 $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
-    [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
-    chmod -R g=u $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
-    [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
-    chmod -R g+w $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
-    [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
-    chmod -R g+rX $PRISMA_PATH && \
-    chmod -R g+rX /app/.cache && \
-    mkdir -p /tmp/.npm /nonexistent /.npm
+COPY --from=builder /app /app
+COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
+COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
 
-# Switch to non-root user for runtime
-USER nobody
-
-# Generate Prisma client as nobody user to ensure correct file ownership
-RUN prisma generate
-
-# Prisma runtime knobs for offline containers
-ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
+ENV PATH="/app/.venv/bin:${PATH}" \
+    PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
+    HOME=/app \
+    LITELLM_NON_ROOT=true \
+    XDG_CACHE_HOME=/app/.cache \
+    PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
     PRISMA_HIDE_UPDATE_MESSAGE=1 \
     PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
-    NPM_CONFIG_CACHE=/app/.cache/npm \
-    NPM_CONFIG_PREFER_OFFLINE=true \
     PRISMA_OFFLINE_MODE=true
 
+RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
+    chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \
+    PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
+    chown -R nobody:nogroup "$PRISMA_PATH" && \
+    LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
+    [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup "$LITELLM_PKG_MIGRATIONS_PATH" || true && \
+    LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
+    chgrp -R 0 "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
+    [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 "$LITELLM_PROXY_EXTRAS_PATH" || true && \
+    chmod -R g=u "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
+    [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u "$LITELLM_PROXY_EXTRAS_PATH" || true && \
+    chmod -R g+w "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets && \
+    [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w "$LITELLM_PROXY_EXTRAS_PATH" || true && \
+    chmod -R g+rX "$PRISMA_PATH" /var/lib/litellm/ui /var/lib/litellm/assets /app/.cache
+
+USER 65534
+
+RUN prisma generate --schema=./schema.prisma
+
 EXPOSE 4000/tcp
+
 ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

docker/README.md

@@ -13,19 +13,19 @@ To build and run the application, you will use the `docker-compose.yml` file loc
 
 ### 1. Set the Master Key
 
-The application requires a `MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
+The application requires a `LITELLM_MASTER_KEY` for signing and validating tokens. You must set this key as an environment variable before running the application.
 
 Create a `.env` file in the root of the project and add the following line:
 
 ```
-MASTER_KEY=your-secret-key
+LITELLM_MASTER_KEY=your-secret-key
 ```
 
 Replace `your-secret-key` with a strong, randomly generated secret.
 
 ### 2. Build and Run the Containers
 
-Once you have set the `MASTER_KEY`, you can build and run the containers using the following command:
+Once you have set the `LITELLM_MASTER_KEY`, you can build and run the containers using the following command:
 
 ```bash
 docker compose up -d --build
@@ -89,4 +89,4 @@ This command should succeed (showing engine versions) even with `--network none`
 ## Troubleshooting
 
 -   **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.
--   **`Master key is not initialized`**: This error means the `MASTER_key` environment variable is not set. Make sure you have created a `.env` file in the project root with the `MASTER_KEY` defined.
+-   **`Master key is not initialized`**: This error means the `LITELLM_MASTER_KEY` environment variable is not set. Make sure you have created a `.env` file in the project root with the `LITELLM_MASTER_KEY` defined.