Merge branch 'main' into litellm_bedrock-embedded-region-model-path

.circleci/requirements.txt

@@ -8,11 +8,14 @@ redis==5.2.1
 redisvl==0.4.1
 anthropic
 orjson==3.10.12 # fast /embedding responses
-pydantic==2.10.2
+pydantic==2.11.0
 google-cloud-aiplatform==1.43.0
 google-cloud-iam==2.19.1
 fastapi-sso==0.16.0 
 uvloop==0.21.0
-mcp==1.10.1    # for MCP server  
+mcp==1.25.0    # for MCP server  
 semantic_router==0.1.10 # for auto-routing with litellm
-fastuuid==0.12.0
+fastuuid==0.12.0
+responses==0.25.7  # for proxy client tests
+pytest-retry==1.6.3  # for automatic test retries
+litellm-proxy-extras  # for prisma migrations

.devcontainer/devcontainer.json

@@ -11,7 +11,12 @@
 	// },
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
-	// "features": {},
+	"features": {
+		"ghcr.io/devcontainers/features/node:1": {
+			"version": "lts"
+		},
+		"ghcr.io/devcontainers/features/docker-in-docker:2": {}
+	},
 
 	// Configure tool-specific properties.
 	"customizations": {
@@ -30,7 +35,7 @@
 	
 	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	"forwardPorts": [4000],
-
+	
 	"containerEnv": {
 		"LITELLM_LOG": "DEBUG"
 	},
@@ -48,5 +53,5 @@
 	// "remoteUser": "litellm",
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy"
+	"postCreateCommand": "bash ./.devcontainer/post-create.sh"
 }

.devcontainer/post-create.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -e
+
+echo "[post-create] Installing poetry via pip"
+python -m pip install --upgrade pip
+python -m pip install poetry
+
+echo "[post-create] Installing Python dependencies (poetry)"
+poetry install --with dev --extras proxy
+
+echo "[post-create] Generating Prisma client"
+poetry run prisma generate
+
+echo "[post-create] Installing npm dependencies"
+cd ui/litellm-dashboard && npm ci
+
+echo "[post-create] Done"

.dockerignore

@@ -4,9 +4,51 @@ cookbook
 .github
 tests
 .git
-.github
-.circleci
 .devcontainer
 *.tgz
 log.txt
 docker/Dockerfile.*
+
+# Claude Flow generated files (must be excluded from Docker build)
+.claude/
+.claude-flow/
+.swarm/
+.hive-mind/
+memory/
+coordination/
+claude-flow
+.mcp.json
+hive-mind-prompt-*.txt
+
+# Python virtual environments and version managers
+.venv/
+venv/
+**/.venv/
+**/venv/
+.python-version
+.pyenv/
+__pycache__/
+**/__pycache__/
+*.pyc
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+**/pyvenv.cfg
+
+# Common project exclusions
+.vscode
+*.pyo
+*.pyd
+.Python
+env/
+.pytest_cache
+.coverage
+htmlcov/
+dist/
+build/
+*.egg-info/
+.DS_Store
+**/node_modules
+*.log
+.env
+.env.local

.gitguardian.yaml

@@ -0,0 +1,111 @@
+version: 2
+
+secret:
+  # Exclude files and paths by globbing
+  ignored_paths:
+    - "**/*.whl"
+    - "**/*.pyc"
+    - "**/__pycache__/**"
+    - "**/node_modules/**"
+    - "**/dist/**"
+    - "**/build/**"
+    - "**/.git/**"
+    - "**/venv/**"
+    - "**/.venv/**"
+    
+    # Large data/metadata files that don't need scanning
+    - "**/model_prices_and_context_window*.json"
+    - "**/*_metadata/*.txt"
+    - "**/tokenizers/*.json"
+    - "**/tokenizers/*"
+    - "miniconda.sh"
+    
+    # Build outputs and static assets
+    - "litellm/proxy/_experimental/out/**"
+    - "ui/litellm-dashboard/public/**"
+    - "**/swagger/*.js"
+    - "**/*.woff"
+    - "**/*.woff2"
+    - "**/*.avif"
+    - "**/*.webp"
+    
+    # Test data files
+    - "**/tests/**/data_map.txt"
+    - "tests/**/*.txt"
+    
+    # Documentation and other non-code files
+    - "docs/**"
+    - "**/*.md"
+    - "**/*.lock"
+    - "poetry.lock"
+    - "package-lock.json"
+
+  # Ignore security incidents with the SHA256 of the occurrence (false positives)
+  ignored_matches:
+    # === Current detected false positives (SHA-based) ===
+    
+    # gcs_pub_sub_body - folder name, not a password
+    - name: GCS pub/sub test folder name
+      match: 75f377c456eede69e5f6e47399ccee6016a2a93cc5dd11db09cc5b1359ae569a
+    
+    # os.environ/APORIA_API_KEY_1 - environment variable reference
+    - name: Environment variable reference APORIA_API_KEY_1
+      match: e2ddeb8b88eca97a402559a2be2117764e11c074d86159ef9ad2375dea188094
+    
+    # os.environ/APORIA_API_KEY_2 - environment variable reference
+    - name: Environment variable reference APORIA_API_KEY_2
+      match: 09aa39a29e050b86603aa55138af1ff08fb86a4582aa965c1bd0672e1575e052
+    
+    # oidc/circleci_v2/ - test authentication path, not a secret
+    - name: OIDC CircleCI test path
+      match: feb3475e1f89a65b7b7815ac4ec597e18a9ec1847742ad445c36ca617b536e15
+    
+    # text-davinci-003 - OpenAI model identifier, not a secret
+    - name: OpenAI model identifier text-davinci-003
+      match: c489000cf6c7600cee0eefb80ad0965f82921cfb47ece880930eb7e7635cf1f1
+    
+    # Base64 Basic Auth in test_pass_through_endpoints.py - test fixture, not a real secret
+    - name: Test Base64 Basic Auth header in pass_through_endpoints test
+      match: 61bac0491f395040617df7ef6d06029eac4d92a4457ac784978db80d97be1ae0
+    
+    # PostgreSQL password "postgres" in CI configs - standard test database password
+    - name: Test PostgreSQL password in CI configurations
+      match: 6e0d657eb1f0fbc40cf0b8f3c3873ef627cc9cb7c4108d1c07d979c04bc8a4bb
+    
+    # Bearer token in locustfile.py - test/example API key for load testing
+    - name: Test Bearer token in locustfile load test
+      match: 2a0abc2b0c3c1760a51ffcdf8d6b1d384cef69af740504b1cfa82dd70cdc7ff9
+    
+    # Inkeep API key in docusaurus.config.js - public documentation site key
+    - name: Inkeep API key in documentation config
+      match: c366657791bfb5fc69045ec11d49452f09a0aebbc8648f94e2469b4025e29a75
+    
+    # Langfuse credentials in test_completion.py - test credentials for integration test
+    - name: Langfuse test credentials in test_completion
+      match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
+    
+    # Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
+    - name: Test password in e2e test fixtures
+      match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
+    
+    # === Preventive patterns for test keys (pattern-based) ===
+    
+    # Test API keys (124 instances across 45 files)
+    - name: Test API keys with sk-test prefix
+      match: sk-test-
+    
+    # Mock API keys
+    - name: Mock API keys with sk-mock prefix
+      match: sk-mock-
+    
+    # Fake API keys
+    - name: Fake API keys with sk-fake prefix
+      match: sk-fake-
+    
+    # Generic test API key patterns
+    - name: Test API key patterns
+      match: test-api-key
+
+    - name: Short fake sk keys (1–9 digits only)
+      match: \bsk-\d{1,9}\b
+

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -7,6 +7,16 @@ body:
     attributes:
       value: |
         Thanks for taking the time to fill out this bug report!
+        
+        **💡 Tip:** See our [Troubleshooting Guide](https://docs.litellm.ai/docs/troubleshoot) for what information to include.
+  - type: checkboxes
+    id: duplicate-check
+    attributes:
+      label: Check for existing issues
+      description: Please search to see if an issue already exists for the bug you encountered.
+      options:
+        - label: I have searched the existing issues and checked that my issue is not a duplicate.
+          required: true
   - type: textarea
     id: what-happened
     attributes:
@@ -16,6 +26,21 @@ body:
       value: "A bug happened!"
     validations:
       required: true
+  - type: textarea
+    id: steps-to-reproduce
+    attributes:
+      label: Steps to Reproduce
+      description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
+      placeholder: |
+        1. config.yaml file/ .env file/ etc.
+        2. Run the following code...
+        3. Observe the error...
+      value: |
+        1. 
+        2. 
+        3. 
+    validations:
+      required: true
   - type: textarea
     id: logs
     attributes:
@@ -23,13 +48,16 @@ body:
       description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
       render: shell
   - type: dropdown
-    id: ml-ops-team
+    id: component
     attributes:
-      label: Are you a ML Ops Team?
-      description: This helps us prioritize your requests correctly 
+      label: What part of LiteLLM is this about?
       options:
-        - "No"
-        - "Yes"
+        - ''
+        - "SDK (litellm Python package)"
+        - "Proxy"
+        - "UI Dashboard"
+        - "Docs"
+        - "Other"
     validations:
       required: true
   - type: input

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,7 @@
 blank_issues_enabled: true
 contact_links:
   - name: Schedule Demo
-    url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat
+    url: https://enterprise.litellm.ai/demo
     about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
   - name: Discord
     url: https://discord.com/invite/wuPM9dRgDw

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -7,6 +7,14 @@ body:
     attributes:
       value: |
         Thanks for making LiteLLM better! 
+  - type: checkboxes
+    id: duplicate-check
+    attributes:
+      label: Check for existing issues
+      description: Please search to see if an issue already exists for the feature you are requesting.
+      options:
+        - label: I have searched the existing issues and checked that my issue is not a duplicate.
+          required: true
   - type: textarea
     id: the-feature
     attributes:
@@ -22,6 +30,19 @@ body:
       description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too.
     validations:
       required: true
+  - type: dropdown
+    id: component
+    attributes:
+      label: What part of LiteLLM is this about?
+      options:
+        - ''
+        - "SDK (litellm Python package)"
+        - "Proxy"
+        - "UI Dashboard"
+        - "Docs"
+        - "Other"
+    validations:
+      required: true
   - type: dropdown
     id: hiring-interest
     attributes:

.github/actions/helm-oci-chart-releaser/action.yml

@@ -40,38 +40,55 @@ outputs:
 runs:
   using: composite
   steps:
+    - name: Helm | Setup
+      uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
+      with:
+        version: v3.20.0
+
     - name: Helm | Login
       shell: bash
-      run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }}
       env:
-        HELM_EXPERIMENTAL_OCI: '1'
-    
+        REGISTRY_PASSWORD: ${{ inputs.registry_password }}
+        REGISTRY_USERNAME: ${{ inputs.registry_username }}
+        REGISTRY: ${{ inputs.registry }}
+      run: echo "$REGISTRY_PASSWORD" | helm registry login -u "$REGISTRY_USERNAME" --password-stdin "$REGISTRY"
+
     - name: Helm | Dependency
       if: inputs.update_dependencies == 'true'
       shell: bash
-      run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
       env:
-        HELM_EXPERIMENTAL_OCI: '1'
+        CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
+      run: helm dependency update "$CHART_PATH"
 
     - name: Helm | Package
       shell: bash
-      run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }}
       env:
-        HELM_EXPERIMENTAL_OCI: '1'
+        CHART_PATH: ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
+        TAG: ${{ inputs.tag }}
+        APP_VERSION: ${{ inputs.app_version }}
+      run: helm package "$CHART_PATH" --version "$TAG" --app-version "$APP_VERSION"
 
     - name: Helm | Push
       shell: bash
-      run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }}
       env:
-        HELM_EXPERIMENTAL_OCI: '1'
+        NAME: ${{ inputs.name }}
+        TAG: ${{ inputs.tag }}
+        REGISTRY: ${{ inputs.registry }}
+        REPOSITORY: ${{ inputs.repository }}
+      run: helm push "${NAME}-${TAG}.tgz" "oci://${REGISTRY}/${REPOSITORY}"
 
     - name: Helm | Logout
       shell: bash
-      run: helm registry logout ${{ inputs.registry }}
       env:
-        HELM_EXPERIMENTAL_OCI: '1'
+        REGISTRY: ${{ inputs.registry }}
+      run: helm registry logout "$REGISTRY"
 
     - name: Helm | Output
       id: output
       shell: bash
-      run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
+      env:
+        REGISTRY: ${{ inputs.registry }}
+        REPOSITORY: ${{ inputs.repository }}
+        NAME: ${{ inputs.name }}
+        TAG: ${{ inputs.tag }}
+      run: echo "image=${REGISTRY}/${REPOSITORY}/${NAME}:${TAG}" >> $GITHUB_OUTPUT

.github/codeql/codeql-config.yml

@@ -0,0 +1,21 @@
+name: "LiteLLM CodeQL config"
+
+queries:
+  - uses: security-and-quality
+
+# Known OOM queries on large Python codebases:
+# CodeQL builds a full data flow graph in memory. These two queries trace
+# sensitive data through every log call / regex pattern, causing combinatorial
+# path explosion on codebases with extensive logging like LiteLLM (>2 GiB
+# result sets). This is a known CodeQL scaling limitation, not a code issue.
+# Re-test periodically as CodeQL improves or the codebase refactors logging.
+query-filters:
+  - exclude:
+      id: py/clear-text-logging-sensitive-data  # CWE-312
+  - exclude:
+      id: py/polynomial-redos                   # CWE-730
+
+paths-ignore:
+  - tests
+  - docs
+  - "**/*.md"

.github/dependabot.yaml

@@ -4,6 +4,9 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    cooldown:
+      default-days: 7
+      semver-major-days: 14
     groups:
       github-actions:
         patterns:

.github/observatory/litellm_config.yaml

@@ -0,0 +1,19 @@
+# LiteLLM Observatory Test Configuration
+# This config is used by CI to spin up a temporary LiteLLM instance
+# for running observatory tests against RC/stable releases.
+#
+# Add model definitions for the providers you want to test.
+# Provider API keys are injected via environment variables in CI.
+
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: azure/gpt-4o
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: azure/gpt-4o-mini
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE

.github/pull_request_template.md

@@ -1,7 +1,3 @@
-## Title
-
-<!-- e.g. "Implement user authentication feature" -->
-
 ## Relevant issues
 
 <!-- e.g. "Fixes #000" -->
@@ -10,11 +6,31 @@
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
-- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
-- [ ] I have added a screenshot of my new test passing locally 
+- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
+- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
 
+## Delays in PR merge?
+
+If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slack (#pr-review)](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA).
+
+## CI (LiteLLM team)
+
+> **CI status guideline:**
+>
+> - 50-55 passing tests: main is stable with minor issues.
+> - 45-49 passing tests: acceptable but needs attention
+> - <= 40 passing tests: unstable; be careful with your merges and assess the risk.
+
+- [ ] **Branch creation CI run**  
+       Link:
+
+- [ ] **CI run for the last commit**  
+       Link:
+
+- [ ] **Merge / cherry-pick CI run**  
+       Links:
 
 ## Type
 
@@ -29,5 +45,3 @@
 ✅ Test
 
 ## Changes
-
-

.github/scripts/close_duplicate_issues.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""
+Detect and close duplicate GitHub issues using title similarity.
+
+Modes:
+  --scan           Compare all open issues against each other (batch)
+  --issue-number N Check a single issue against older open issues
+
+Requires the `gh` CLI to be authenticated.
+"""
+
+import argparse
+import difflib
+import json
+import re
+import subprocess
+import sys
+
+
+def normalize_title(title: str) -> str:
+    """Strip common prefixes, lowercase, and collapse whitespace."""
+    title = re.sub(
+        r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
+        "",
+        title,
+        flags=re.IGNORECASE,
+    )
+    return " ".join(title.lower().split())
+
+
+def gh(*args: str) -> str:
+    """Run a gh CLI command and return stdout."""
+    result = subprocess.run(
+        ["gh", *args],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def fetch_open_issues(repo: str | None) -> list[dict]:
+    """Fetch all open issues (excluding PRs) via gh api --paginate."""
+    if repo:
+        endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    else:
+        endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    cmd = ["api", "--paginate", endpoint]
+
+    raw = gh(*cmd)
+    # gh --paginate concatenates JSON arrays, so we may get multiple arrays
+    issues = []
+    for line in raw.strip().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parsed = json.loads(line)
+        if isinstance(parsed, list):
+            issues.extend(parsed)
+        else:
+            issues.append(parsed)
+
+    # Filter out pull requests (they also appear in the issues endpoint)
+    return [i for i in issues if "pull_request" not in i]
+
+
+def close_as_duplicate(
+    issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
+) -> None:
+    """Close an issue as duplicate of another, adding a comment and label."""
+    repo_args = ["--repo", repo] if repo else []
+
+    if dry_run:
+        print(f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
+        return
+
+    # Add comment
+    comment_body = (
+        f"Closing as duplicate of #{duplicate_of}.\n\n"
+        "If you believe this is not a duplicate, please reopen and add context "
+        "explaining how this differs."
+    )
+    gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
+
+    # Add label
+    gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
+
+    # Close with not_planned reason
+    gh(
+        "api",
+        f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
+        "-X",
+        "PATCH",
+        "-f",
+        "state=closed",
+        "-f",
+        "state_reason=not_planned",
+    )
+
+    print(f"  Closed #{issue_number} as duplicate of #{duplicate_of}")
+
+
+def find_duplicate(
+    issue: dict, candidates: list[dict], threshold: float
+) -> dict | None:
+    """Return the first candidate whose normalized title is above threshold."""
+    norm = normalize_title(issue["title"])
+    for candidate in candidates:
+        if candidate["number"] == issue["number"]:
+            continue
+        cand_norm = normalize_title(candidate["title"])
+        ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
+        if ratio >= threshold:
+            return candidate
+    return None
+
+
+def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
+    """Compare every issue against all older issues. Returns count of duplicates found."""
+    # Sort oldest first
+    issues.sort(key=lambda i: i["number"])
+    closed_count = 0
+
+    for idx, issue in enumerate(issues):
+        older = issues[:idx]
+        if not older:
+            continue
+        dup = find_duplicate(issue, older, threshold)
+        if dup:
+            ratio = difflib.SequenceMatcher(
+                None,
+                normalize_title(issue["title"]),
+                normalize_title(dup["title"]),
+            ).ratio()
+            print(
+                f"#{issue['number']}: \"{issue['title']}\"\n"
+                f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+                f"({ratio:.0%} similar)"
+            )
+            close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
+            closed_count += 1
+
+    return closed_count
+
+
+def check_single(
+    issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+) -> bool:
+    """Check a single issue against all older open issues. Returns True if duplicate found."""
+    target = None
+    for i in issues:
+        if i["number"] == issue_number:
+            target = i
+            break
+
+    if target is None:
+        print(f"Issue #{issue_number} not found among open issues.")
+        return False
+
+    older = [i for i in issues if i["number"] < issue_number]
+    dup = find_duplicate(target, older, threshold)
+    if dup:
+        ratio = difflib.SequenceMatcher(
+            None,
+            normalize_title(target["title"]),
+            normalize_title(dup["title"]),
+        ).ratio()
+        print(
+            f"#{target['number']}: \"{target['title']}\"\n"
+            f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+            f"({ratio:.0%} similar)"
+        )
+        close_as_duplicate(issue_number, dup["number"], repo, dry_run)
+        return True
+
+    print(f"#{issue_number}: no duplicate found above threshold {threshold}")
+    return False
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--scan", action="store_true", help="Scan all open issues")
+    mode.add_argument("--issue-number", type=int, help="Check a single issue number")
+    parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
+    parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
+    parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
+    args = parser.parse_args()
+
+    dry_run = not args.close
+
+    if dry_run:
+        print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
+
+    print("Fetching open issues...")
+    issues = fetch_open_issues(args.repo)
+    print(f"Found {len(issues)} open issues.\n")
+
+    if args.scan:
+        count = scan_all(issues, args.threshold, args.repo, dry_run)
+        print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
+    else:
+        found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
+        sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/_test-unit-base.yml

@@ -0,0 +1,96 @@
+name: _Unit Test Base (Reusable)
+
+on:
+  workflow_call:
+    inputs:
+      test-path:
+        description: "Pytest path(s) to run"
+        required: true
+        type: string
+      workers:
+        description: "Number of pytest-xdist workers"
+        required: false
+        type: number
+        default: 2
+      reruns:
+        description: "Number of reruns for flaky tests"
+        required: false
+        type: number
+        default: 2
+      timeout-minutes:
+        description: "Job timeout in minutes"
+        required: false
+        type: number
+        default: 20
+      max-failures:
+        description: "Stop after this many failures"
+        required: false
+        type: number
+        default: 10
+
+permissions:
+  contents: read
+
+jobs:
+  run:
+    name: Run tests
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ inputs.timeout-minutes }}
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          poetry run pip install google-genai==1.22.0 \
+            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          poetry run pip install nodejs-wheel-binaries==24.13.1
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run tests
+        env:
+          TEST_PATH: ${{ inputs.test-path }}
+          MAX_FAILURES: ${{ inputs.max-failures }}
+          WORKERS: ${{ inputs.workers }}
+          RERUNS: ${{ inputs.reruns }}
+        run: |
+          poetry run pytest ${TEST_PATH:?} \
+            --tb=short -vv \
+            --maxfail="${MAX_FAILURES}" \
+            -n "${WORKERS}" \
+            --reruns "${RERUNS}" \
+            --reruns-delay 1 \
+            --dist=loadscope \
+            --durations=20

.github/workflows/_test-unit-services-base.yml

@@ -0,0 +1,164 @@
+name: _Unit Test Services Base (Reusable)
+
+on:
+  workflow_call:
+    inputs:
+      test-path:
+        description: "Pytest path(s) to run"
+        required: true
+        type: string
+      workers:
+        description: "Number of pytest-xdist workers (0 = no parallelism)"
+        required: false
+        type: number
+        default: 2
+      reruns:
+        description: "Number of reruns for flaky tests"
+        required: false
+        type: number
+        default: 2
+      timeout-minutes:
+        description: "Job timeout in minutes"
+        required: false
+        type: number
+        default: 20
+      max-failures:
+        description: "Stop after this many failures"
+        required: false
+        type: number
+        default: 10
+      enable-redis:
+        description: "Pass Redis Cloud credentials to tests via REDIS_HOST/PORT/PASSWORD env vars"
+        required: false
+        type: boolean
+        default: false
+      enable-postgres:
+        description: "Start a local Postgres service container and run Prisma migrations"
+        required: false
+        type: boolean
+        default: false
+    secrets:
+      REDIS_HOST:
+        required: false
+      REDIS_PORT:
+        required: false
+      REDIS_PASSWORD:
+        required: false
+      DATABASE_URL:
+        required: false
+      POSTGRES_USER:
+        required: false
+      POSTGRES_PASSWORD:
+        required: false
+
+permissions:
+  contents: read
+
+jobs:
+  run:
+    name: Run tests
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ inputs.timeout-minutes }}
+    # Environment is derived from the enable-* flags, not caller-controllable.
+    # This prevents callers from passing arbitrary environment names to bypass secret scoping.
+    # Note: Postgres service container always starts (GHA limitation), so any Redis job
+    # also needs Postgres secrets → uses integration-redis-postgres, not integration-redis.
+    environment: >-
+      ${{
+        inputs.enable-redis && 'integration-redis-postgres' ||
+        inputs.enable-postgres && 'integration-postgres' ||
+        ''
+      }}
+
+    services:
+      postgres:
+        image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14
+        env:
+          POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+          POSTGRES_DB: litellm_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-services-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-services-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          poetry run pip install google-genai==1.22.0 \
+            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          poetry run pip install nodejs-wheel-binaries==24.13.1
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run Prisma migrations
+        if: ${{ inputs.enable-postgres }}
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+        run: |
+          poetry run prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+
+      - name: Run tests
+        env:
+          TEST_PATH: ${{ inputs.test-path }}
+          MAX_FAILURES: ${{ inputs.max-failures }}
+          WORKERS: ${{ inputs.workers }}
+          RERUNS: ${{ inputs.reruns }}
+          DATABASE_URL: ${{ inputs.enable-postgres && secrets.DATABASE_URL || '' }}
+          REDIS_HOST: ${{ inputs.enable-redis && secrets.REDIS_HOST || '' }}
+          REDIS_PORT: ${{ inputs.enable-redis && secrets.REDIS_PORT || '' }}
+          REDIS_PASSWORD: ${{ inputs.enable-redis && secrets.REDIS_PASSWORD || '' }}
+        run: |
+          if [ "${WORKERS}" = "0" ]; then
+            poetry run pytest ${TEST_PATH:?} \
+              --tb=short -vv \
+              --maxfail="${MAX_FAILURES}" \
+              --reruns "${RERUNS}" \
+              --reruns-delay 1 \
+              --durations=20
+          else
+            poetry run pytest ${TEST_PATH:?} \
+              --tb=short -vv \
+              --maxfail="${MAX_FAILURES}" \
+              -n "${WORKERS}" \
+              --reruns "${RERUNS}" \
+              --reruns-delay 1 \
+              --dist=loadscope \
+              --durations=20
+          fi

.github/workflows/auto_update_price_and_context_window.yml

@@ -2,17 +2,24 @@ name: Updates model_prices_and_context_window.json and Create Pull Request
 
 on:
   schedule:
-    - cron: "0 0 * * 0"  # Run every Sundays at midnight
+    - cron: "0 0 * * 0" # Run every Sundays at midnight
     #- cron: "0 0 * * *" # Run daily at midnight
 
+permissions:
+  contents: write
+  pull-requests: write
+
 jobs:
   auto_update_price_and_context_window:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
       - name: Install Dependencies
         run: |
-          pip install aiohttp
+          pip install 'aiohttp==3.13.3'
       - name: Update JSON Data
         run: |
           python ".github/workflows/auto_update_price_and_context_window_file.py"
@@ -25,4 +32,4 @@ jobs:
             --head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \
             --base main
         env:
-          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}

.github/workflows/check-schema-sync.yml

@@ -0,0 +1,58 @@
+name: Check Schema Sync
+
+on:
+  pull_request:
+    paths:
+      - 'schema.prisma'
+      - 'litellm/proxy/schema.prisma'
+      - 'litellm-proxy-extras/litellm_proxy_extras/schema.prisma'
+
+permissions:
+  contents: read
+
+jobs:
+  check-sync:
+    name: Verify schema.prisma copies match root
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Checkout PR
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Reject symlinked schema files
+        run: |
+          for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do
+            if [ -L "$f" ]; then
+              echo "::error file=$f::$f is a symlink, which is not allowed"
+              exit 1
+            fi
+          done
+
+      - name: Check all schemas match root
+        run: |
+          EXIT=0
+
+          diff schema.prisma litellm/proxy/schema.prisma || {
+            echo "::error file=litellm/proxy/schema.prisma::litellm/proxy/schema.prisma differs from root schema.prisma"
+            EXIT=1
+          }
+
+          diff schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma || {
+            echo "::error file=litellm-proxy-extras/litellm_proxy_extras/schema.prisma::litellm-proxy-extras/litellm_proxy_extras/schema.prisma differs from root schema.prisma"
+            EXIT=1
+          }
+
+          if [ "$EXIT" -ne 0 ]; then
+            echo ""
+            echo "Schema files are out of sync."
+            echo "The root schema.prisma is the source of truth."
+            echo ""
+            echo "To fix, run from the repo root:"
+            echo "  cp schema.prisma litellm/proxy/schema.prisma"
+            echo "  cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma"
+            exit 1
+          fi
+
+          echo "All schema copies are in sync with root."

.github/workflows/check_duplicate_issues.yml

@@ -0,0 +1,53 @@
+name: Check Duplicate Issues
+
+on:
+  issues:
+    types: [opened, edited]
+
+jobs:
+  check-duplicate:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Check for potential duplicates
+        uses: wow-actions/potential-duplicates@4d4ea0352e0383859279938e255179dd1dbb67b5 # v1.1.0
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          label: potential-duplicate
+          threshold: 0.6
+          reaction: eyes
+          comment: |
+            **⚠️ Potential duplicate detected**
+
+            This issue appears similar to existing issue(s):
+            {{#issues}}
+            - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
+            {{/issues}}
+
+            Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
+
+      - name: Checkout close script
+        if: github.event.action == 'opened'
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          sparse-checkout: .github/scripts
+          persist-credentials: false
+
+      - name: Set up Python
+        if: github.event.action == 'opened'
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.11"
+
+      - name: Auto-close if high-confidence duplicate
+        if: github.event.action == 'opened'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 .github/scripts/close_duplicate_issues.py \
+            --issue-number ${{ github.event.issue.number }} \
+            --repo ${{ github.repository }} \
+            --threshold 0.85 \
+            --close

.github/workflows/codeql.yml

@@ -0,0 +1,55 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    # Run daily at 04:00 UTC
+    - cron: "0 4 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  analyze:
+    if: github.event_name != 'schedule' || github.repository == 'BerriAI/litellm'
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: actions
+            build-mode: none
+          - language: javascript-typescript
+            build-mode: none
+          - language: python
+            build-mode: none
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          config-file: ./.github/codeql/codeql-config.yml
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@ebcb5b36ded6beda4ceefea6a8bc4cc885255bb3 # v3
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/codspeed.yml

@@ -0,0 +1,46 @@
+name: CodSpeed Benchmarks
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  # Allow CodSpeed to trigger backtest performance analysis
+  # in order to generate initial data
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          pip install -e "."
+          pip install pytest pytest-codspeed==4.3.0
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
+        with:
+          mode: simulation
+          run: pytest tests/benchmarks/ --codspeed

.github/workflows/create-release.yml

@@ -0,0 +1,93 @@
+name: Create Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag (e.g. v1.83.0-stable)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Full 40-char commit SHA to target"
+        required: true
+        type: string
+
+permissions: {}
+
+jobs:
+  release:
+    name: Create Release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Validate inputs
+        env:
+          TAG: ${{ inputs.tag }}
+          COMMIT_HASH: ${{ inputs.commit_hash }}
+        run: |
+          if ! echo "${COMMIT_HASH}" | grep -qE '^[0-9a-f]{40}$'; then
+            echo "::error::commit_hash must be a full 40-character commit SHA"
+            exit 1
+          fi
+          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with vX.Y.Z"
+            exit 1
+          fi
+
+      - name: Create release
+        env:
+          TAG: ${{ inputs.tag }}
+          COMMIT_HASH: ${{ inputs.commit_hash }}
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            const tag = process.env.TAG;
+            const commitHash = process.env.COMMIT_HASH;
+
+            const cosignSection = [
+              `## Verify Docker Image Signature`,
+              ``,
+              `All LiteLLM Docker images are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). To verify the integrity of an image before deploying:`,
+              ``,
+              '```bash',
+              `cosign verify \\`,
+              `  --key https://raw.githubusercontent.com/BerriAI/litellm/${tag}/cosign.pub \\`,
+              `  ghcr.io/berriai/litellm:${tag}`,
+              '```',
+              ``,
+              `Expected output:`,
+              ``,
+              '```',
+              `The following checks were performed on each of these signatures:`,
+              `  - The cosign claims were validated`,
+              `  - The signatures were verified against the specified public key`,
+              '```',
+              ``,
+              `---`,
+              ``,
+            ].join('\n');
+
+            try {
+              const response = await github.rest.repos.createRelease({
+                draft: true,
+                generate_release_notes: true,
+                target_commitish: commitHash,
+                name: tag,
+                owner: context.repo.owner,
+                prerelease: false,
+                repo: context.repo.repo,
+                tag_name: tag,
+              });
+
+              const updatedBody = cosignSection + (response.data.body ?? '');
+              await github.rest.repos.updateRelease({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                release_id: response.data.id,
+                body: updatedBody,
+                draft: false,
+              });
+            } catch (error) {
+              core.setFailed(error.message);
+            }

.github/workflows/create_daily_staging_branch.yml

@@ -0,0 +1,87 @@
+name: Create Daily Staging Branch
+
+on:
+  schedule:
+    - cron: "0 0,12 * * *" # Runs every 12 hours at midnight and noon UTC
+  workflow_dispatch: # Allow manual trigger
+
+jobs:
+  create-staging-branch:
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Create daily staging branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Configure Git user
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Generate branch name with MM_DD_YYYY format
+          BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
+          echo "Creating branch: $BRANCH_NAME"
+
+          # Fetch all branches
+          git fetch --all
+
+          # Check if the branch already exists
+          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+            echo "Branch $BRANCH_NAME already exists. Skipping creation."
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            # Create the new branch from main
+            git checkout -b $BRANCH_NAME origin/main
+            # Push the new branch
+            git push origin $BRANCH_NAME
+            echo "Successfully created and pushed branch: $BRANCH_NAME"
+          fi
+
+  create-internal-dev-branch:
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Create internal dev branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Configure Git user
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Generate branch name with MM_DD_YYYY format
+          BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
+          echo "Creating branch: $BRANCH_NAME"
+
+          # Fetch all branches
+          git fetch --all
+
+          # Check if the branch already exists
+          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+            echo "Branch $BRANCH_NAME already exists. Skipping creation."
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            # Create the new branch from main
+            git checkout -b $BRANCH_NAME origin/main
+            # Push the new branch
+            git push origin $BRANCH_NAME
+            echo "Successfully created and pushed branch: $BRANCH_NAME"
+          fi

.github/workflows/ghcr_deploy.yml

@@ -1,440 +0,0 @@
-# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
-name: Build, Publish LiteLLM Docker Image. New Release
-on:
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: "The tag version you want to build"
-      release_type:
-        description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
-        type: string
-        default: "latest"
-      commit_hash:
-        description: "Commit hash"
-        required: true
-
-# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
-  CHART_NAME: litellm-helm
-
-# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
-jobs:
-  # print commit hash, tag, and release type
-  print:
-    runs-on: ubuntu-latest
-    steps:
-      - run: |
-          echo "Commit hash: ${{ github.event.inputs.commit_hash }}"
-          echo "Tag: ${{ github.event.inputs.tag }}"
-          echo "Release type: ${{ github.event.inputs.release_type }}"
-  docker-hub-deploy:
-    if: github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.commit_hash }}
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      -
-        name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      -
-        name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          tags: litellm/litellm:${{ github.event.inputs.tag || 'latest' }} 
-      -
-        name: Build and push litellm-database image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          file: ./docker/Dockerfile.database
-          tags: litellm/litellm-database:${{ github.event.inputs.tag || 'latest' }}
-      -
-        name: Build and push litellm-spend-logs image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          file: ./litellm-js/spend-logs/Dockerfile
-          tags: litellm/litellm-spend_logs:${{ github.event.inputs.tag || 'latest' }}
-      -
-        name: Build and push litellm-non_root image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          file: ./docker/Dockerfile.non_root
-          tags: litellm/litellm-non_root:${{ github.event.inputs.tag || 'latest' }}
-  build-and-push-image:
-    runs-on: ubuntu-latest
-    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.commit_hash }}
-      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels.
-      - name: Extract metadata (tags, labels) for Docker
-        id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-      # Configure multi platform Docker builds
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
-      # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
-      # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
-      # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
-      - name: Build and push Docker image
-        uses: docker/build-push-action@4976231911ebf5f32aad765192d35f942aa48cb8
-        with:
-          context: .
-          push: true
-          tags: |
-            ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
-            ${{ steps.meta.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm:main-stable', env.REGISTRY) || '' }},
-            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm:{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
-          labels: ${{ steps.meta.outputs.labels }}
-          platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-  
-  build-and-push-image-ee:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.commit_hash }}
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for EE Dockerfile
-        id: meta-ee
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
-      # Configure multi platform Docker builds
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
-
-      - name: Build and push EE Docker image
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
-        with:
-          context: .
-          file: Dockerfile
-          push: true
-          tags: |
-            ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
-            ${{ steps.meta-ee.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-ee:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-ee:main-stable', env.REGISTRY) || '' }}
-          labels: ${{ steps.meta-ee.outputs.labels }}
-          platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-   
-  build-and-push-image-database:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.commit_hash }}
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for database Dockerfile
-        id: meta-database
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-database
-      # Configure multi platform Docker builds
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
-
-      - name: Build and push Database Docker image
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
-        with:
-          context: .
-          file: ./docker/Dockerfile.database
-          push: true
-          tags: |
-            ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
-            ${{ steps.meta-database.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-database:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-database:main-stable', env.REGISTRY) || '' }}
-          labels: ${{ steps.meta-database.outputs.labels }}
-          platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-            
-  build-and-push-image-non_root:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.commit_hash }}
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for non_root Dockerfile
-        id: meta-non_root
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-non_root
-      # Configure multi platform Docker builds
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
-
-      - name: Build and push non_root Docker image
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
-        with:
-          context: .
-          file: ./docker/Dockerfile.non_root
-          push: true
-          tags: |
-            ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
-            ${{ steps.meta-non_root.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-non_root:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-non_root:main-stable', env.REGISTRY) || '' }}
-          labels: ${{ steps.meta-non_root.outputs.labels }} 
-          platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-  
-  build-and-push-image-spend-logs:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.inputs.commit_hash }}
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Extract metadata (tags, labels) for spend-logs Dockerfile
-        id: meta-spend-logs
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-spend_logs
-      # Configure multi platform Docker builds
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@e0e4588fad221d38ee467c0bffd91115366dc0c5
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@edfb0fe6204400c56fbfd3feba3fe9ad1adfa345
-
-      - name: Build and push Database Docker image
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
-        with:
-          context: .
-          file: ./litellm-js/spend-logs/Dockerfile
-          push: true
-          tags: |
-            ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.tag || 'latest' }},
-            ${{ steps.meta-spend-logs.outputs.tags }}-${{ github.event.inputs.release_type }}
-            ${{ (github.event.inputs.release_type == 'stable'  || github.event.inputs.release_type == 'rc') && format('{0}/berriai/litellm-spend_logs:main-{1}', env.REGISTRY, github.event.inputs.tag) || '' }},
-            ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
-          platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
-
-  build-and-push-helm-chart:
-    if: github.event.inputs.release_type  != 'dev'
-    needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: lowercase github.repository_owner
-        run: |
-          echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
-    
-      - name: Get LiteLLM Latest Tag
-        id: current_app_tag
-        shell: bash
-        run: |
-          LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0)
-          if [ -z "${LATEST_TAG}" ]; then
-            echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT
-          else
-            echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT
-          fi
-
-      - name: Get last published chart version
-        id: current_version
-        shell: bash
-        run: |
-          CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
-          if [ -z "${CHART_LIST}" ]; then
-            echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
-          else
-            printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
-          fi
-        env:
-          HELM_EXPERIMENTAL_OCI: '1'
-
-      # Automatically update the helm chart version one "patch" level
-      - name: Bump release version
-        id: bump_version
-        uses: christian-draeger/increment-semantic-version@1.1.0
-        with:
-          current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
-          version-fragment: 'bug'
-
-      - uses: ./.github/actions/helm-oci-chart-releaser
-        with:
-          name: ${{ env.CHART_NAME }}
-          repository: ${{ env.REPO_OWNER }}
-          tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
-          app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
-          path: deploy/charts/${{ env.CHART_NAME }}
-          registry: ${{ env.REGISTRY }}
-          registry_username: ${{ github.actor }}
-          registry_password: ${{ secrets.GITHUB_TOKEN }}
-          update_dependencies: true
-
-  release:
-    name: "New LiteLLM Release"
-    needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
-
-    runs-on: "ubuntu-latest"
-   
-    steps:
-      - name: Display version
-        run: echo "Current version is ${{ github.event.inputs.tag }}"
-      - name: "Set Release Tag"
-        run: echo "RELEASE_TAG=${{ github.event.inputs.tag }}" >> $GITHUB_ENV
-      - name: Display release tag
-        run: echo "RELEASE_TAG is $RELEASE_TAG"
-      - name: "Create release"
-        uses: "actions/github-script@v6"
-        with:
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-          script: |
-            const commitHash = "${{ github.event.inputs.commit_hash}}";
-            console.log("Commit Hash:", commitHash); // Add this line for debugging
-            try {
-              const response = await github.rest.repos.createRelease({
-                draft: false,
-                generate_release_notes: true,
-                target_commitish: commitHash,
-                name: process.env.RELEASE_TAG,
-                owner: context.repo.owner,
-                prerelease: false,
-                repo: context.repo.repo,
-                tag_name: process.env.RELEASE_TAG,
-              });
-      
-              core.exportVariable('RELEASE_ID', response.data.id);
-              core.exportVariable('RELEASE_UPLOAD_URL', response.data.upload_url);
-            } catch (error) {
-              core.setFailed(error.message);
-            }
-      - name: Fetch Release Notes
-        id: release-notes
-        uses: actions/github-script@v6
-        with:
-          github-token: "${{ secrets.GITHUB_TOKEN }}"
-          script: |
-            try {
-              const response = await github.rest.repos.getRelease({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                release_id: process.env.RELEASE_ID,
-              });
-              const formattedBody = JSON.stringify(response.data.body).slice(1, -1);
-              return formattedBody;
-            } catch (error) {
-              core.setFailed(error.message);
-            }
-        env:
-          RELEASE_ID: ${{ env.RELEASE_ID }}
-      - name: Github Releases To Discord
-        env:
-          WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }}
-          REALEASE_TAG: ${{ env.RELEASE_TAG }}
-          RELEASE_NOTES: ${{ steps.release-notes.outputs.result }}
-        run: |
-          curl -H "Content-Type: application/json" -X POST -d '{
-            "content": "New LiteLLM release '"${RELEASE_TAG}"'",
-            "username": "Release Changelog",
-            "avatar_url": "https://cdn.discordapp.com/avatars/487431320314576937/bd64361e4ba6313d561d54e78c9e7171.png",
-            "embeds": [
-              {
-                "title": "Changelog for LiteLLM '"${RELEASE_TAG}"'",
-                "description": "'"${RELEASE_NOTES}"'",
-                "color": 2105893
-              }
-            ]
-          }' $WEBHOOK_URL
-

.github/workflows/ghcr_helm_deploy.yml

@@ -1,67 +0,0 @@
-# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
-name: Build, Publish LiteLLM Helm Chart. New Release
-on:
-  workflow_dispatch:
-    inputs:
-      chartVersion:
-        description: "Update the helm chart's version to this"
-
-# Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
-  REPO_OWNER: ${{github.repository_owner}}
-
-# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
-jobs:        
-  build-and-push-helm-chart:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Log in to the Container registry
-        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      
-      - name: lowercase github.repository_owner
-        run: |
-          echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
-
-      - name: Get LiteLLM Latest Tag
-        id: current_app_tag
-        uses: WyriHaximus/github-action-get-previous-tag@v1.3.0
-
-      - name: Get last published chart version
-        id: current_version
-        shell: bash
-        run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
-        env:
-          HELM_EXPERIMENTAL_OCI: '1'
-
-      # Automatically update the helm chart version one "patch" level
-      - name: Bump release version
-        id: bump_version
-        uses: christian-draeger/increment-semantic-version@1.1.0
-        with:
-          current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
-          version-fragment: 'bug'
-
-      - name: Lint helm chart
-        run: helm lint deploy/charts/litellm-helm
-
-      - uses: ./.github/actions/helm-oci-chart-releaser
-        with:
-          name: litellm-helm
-          repository: ${{ env.REPO_OWNER }}
-          tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
-          app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
-          path: deploy/charts/litellm-helm
-          registry: ${{ env.REGISTRY }}
-          registry_username: ${{ github.actor }}
-          registry_password: ${{ secrets.GITHUB_TOKEN }}
-          update_dependencies: true
-  

.github/workflows/helm_unit_test.yml

@@ -6,22 +6,36 @@ on:
     branches:
       - main
 
+permissions:
+  contents: read
+
 jobs:
   unit-test:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
       - name: Set up Helm 3.11.1
-        uses: azure/setup-helm@v1
+        uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
         with:
-          version: '3.11.1'
+          version: "3.11.1"
 
       - name: Install Helm Unit Test Plugin
         run: |
           helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4
+      - name: Verify Helm Unit Test Plugin integrity
+        run: |
+          EXPECTED_SHA="e251ba198448629678ff2168e1a469249d998155"
+          PLUGIN_DIR="$(helm env HELM_PLUGINS)/helm-unittest"
+          ACTUAL_SHA="$(git -C "$PLUGIN_DIR" rev-parse HEAD)"
+          if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
+            echo "::error::Helm unittest plugin checksum mismatch! Expected $EXPECTED_SHA but got $ACTUAL_SHA"
+            exit 1
+          fi
+          echo "Helm unittest plugin integrity verified: $ACTUAL_SHA"
 
       - name: Run unit tests
-        run:
-          helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
+        run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm

.github/workflows/interpret_load_test.py

@@ -1,138 +0,0 @@
-import csv
-import os
-from github import Github
-
-
-def interpret_results(csv_file):
-    with open(csv_file, newline="") as csvfile:
-        csvreader = csv.DictReader(csvfile)
-        rows = list(csvreader)
-        """
-        in this csv reader
-        - Create 1 new column "Status"
-        - if a row has a median response time < 300 and an average response time < 300, Status = "Passed ✅"
-        - if a row has a median response time >= 300 or an average response time >= 300, Status = "Failed ❌"
-        - Order the table in this order Name, Status, Median Response Time, Average Response Time, Requests/s,Failures/s, Min Response Time, Max Response Time, all other columns
-        """
-
-        # Add a new column "Status"
-        for row in rows:
-            median_response_time = float(
-                row["Median Response Time"].strip().rstrip("ms")
-            )
-            average_response_time = float(
-                row["Average Response Time"].strip().rstrip("s")
-            )
-
-            request_count = int(row["Request Count"])
-            failure_count = int(row["Failure Count"])
-
-            failure_percent = round((failure_count / request_count) * 100, 2)
-
-            # Determine status based on conditions
-            if (
-                median_response_time < 300
-                and average_response_time < 300
-                and failure_percent < 5
-            ):
-                row["Status"] = "Passed ✅"
-            else:
-                row["Status"] = "Failed ❌"
-
-        # Construct Markdown table header
-        markdown_table = "| Name | Status | Median Response Time (ms) | Average Response Time (ms) | Requests/s | Failures/s | Request Count | Failure Count | Min Response Time (ms) | Max Response Time (ms) |"
-        markdown_table += (
-            "\n| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"
-        )
-
-        # Construct Markdown table rows
-        for row in rows:
-            markdown_table += f"\n| {row['Name']} | {row['Status']} | {row['Median Response Time']} | {row['Average Response Time']} | {row['Requests/s']} | {row['Failures/s']} | {row['Request Count']} | {row['Failure Count']} | {row['Min Response Time']} | {row['Max Response Time']} |"
-    print("markdown table: ", markdown_table)
-    return markdown_table
-
-
-def _get_docker_run_command_stable_release(release_version):
-    return f"""
-\n\n
-## Docker Run LiteLLM Proxy
-
-```
-docker run \\
--e STORE_MODEL_IN_DB=True \\
--p 4000:4000 \\
-ghcr.io/berriai/litellm:litellm_stable_release_branch-{release_version}
-```
-    """
-
-
-def _get_docker_run_command(release_version):
-    return f"""
-\n\n
-## Docker Run LiteLLM Proxy
-
-```
-docker run \\
--e STORE_MODEL_IN_DB=True \\
--p 4000:4000 \\
-ghcr.io/berriai/litellm:main-{release_version}
-```
-    """
-
-
-def get_docker_run_command(release_version):
-    if "stable" in release_version:
-        return _get_docker_run_command_stable_release(release_version)
-    else:
-        return _get_docker_run_command(release_version)
-
-
-if __name__ == "__main__":
-    csv_file = "load_test_stats.csv"  # Change this to the path of your CSV file
-    markdown_table = interpret_results(csv_file)
-
-    # Update release body with interpreted results
-    github_token = os.getenv("GITHUB_TOKEN")
-    g = Github(github_token)
-    repo = g.get_repo(
-        "BerriAI/litellm"
-    )  # Replace with your repository's username and name
-    latest_release = repo.get_latest_release()
-    print("got latest release: ", latest_release)
-    print(latest_release.title)
-    print(latest_release.tag_name)
-
-    release_version = latest_release.title
-
-    print("latest release body: ", latest_release.body)
-    print("markdown table: ", markdown_table)
-
-    # check if "Load Test LiteLLM Proxy Results" exists
-    existing_release_body = latest_release.body
-    if "Load Test LiteLLM Proxy Results" in latest_release.body:
-        # find the "Load Test LiteLLM Proxy Results" section and delete it
-        start_index = latest_release.body.find("Load Test LiteLLM Proxy Results")
-        existing_release_body = latest_release.body[:start_index]
-
-    docker_run_command = get_docker_run_command(release_version)
-    print("docker run command: ", docker_run_command)
-
-    new_release_body = (
-        existing_release_body
-        + docker_run_command
-        + "\n\n"
-        + "### Don't want to maintain your internal proxy? get in touch 🎉"
-        + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
-        + "\n\n"
-        + "## Load Test LiteLLM Proxy Results"
-        + "\n\n"
-        + markdown_table
-    )
-    print("new release body: ", new_release_body)
-    try:
-        latest_release.update_release(
-            name=latest_release.tag_name,
-            message=new_release_body,
-        )
-    except Exception as e:
-        print(e)

.github/workflows/issue-keyword-labeler.yml

@@ -2,8 +2,8 @@ name: Issue Keyword Labeler
 
 on:
   issues:
-    types: 
-        - opened
+    types:
+      - opened
 
 jobs:
   scan-and-label:
@@ -13,18 +13,20 @@ jobs:
       contents: read
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
       - name: Scan for provider keywords
         id: scan
         env:
           PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
-          KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic
+          KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic,gemini,cohere,mistral,groq,ollama,deepseek
         run: python3 .github/scripts/scan_keywords.py
 
       - name: Ensure label exists
         if: steps.scan.outputs.found == 'true'
-        uses: actions/github-script@v7
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -51,7 +53,7 @@ jobs:
 
       - name: Add label to the issue
         if: steps.scan.outputs.found == 'true'
-        uses: actions/github-script@v7
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -61,4 +63,3 @@ jobs:
               issue_number: context.issue.number,
               labels: ['llm translation']
             });
-

.github/workflows/label-component.yml

@@ -0,0 +1,116 @@
+name: Label Component Issues
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-component-label:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - name: Add component labels
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const body = context.payload.issue.body;
+            if (!body) return;
+
+            // Define component mappings with regex patterns that handle flexible whitespace
+            const components = [
+              {
+                pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/,
+                label: 'sdk',
+                color: '0E7C86',
+                description: 'Issues related to the litellm Python SDK'
+              },
+              {
+                pattern: /What part of LiteLLM is this about\?\s*Proxy/,
+                label: 'proxy',
+                color: '5319E7',
+                description: 'Issues related to the LiteLLM Proxy'
+              },
+              {
+                pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/,
+                label: 'ui-dashboard',
+                color: 'D876E3',
+                description: 'Issues related to the LiteLLM UI Dashboard'
+              },
+              {
+                pattern: /What part of LiteLLM is this about\?\s*Docs/,
+                label: 'docs',
+                color: 'FBCA04',
+                description: 'Issues related to LiteLLM documentation'
+              }
+            ];
+
+            // Find matching component
+            for (const component of components) {
+              if (component.pattern.test(body)) {
+                // Ensure label exists
+                try {
+                  await github.rest.issues.getLabel({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    name: component.label
+                  });
+                } catch (error) {
+                  if (error.status === 404) {
+                    await github.rest.issues.createLabel({
+                      owner: context.repo.owner,
+                      repo: context.repo.repo,
+                      name: component.label,
+                      color: component.color,
+                      description: component.description
+                    });
+                  }
+                }
+
+                // Add label to issue
+                await github.rest.issues.addLabels({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  labels: [component.label]
+                });
+
+                break;
+              }
+            }
+
+            // Check for 'claude code' keyword (can be applied alongside component labels)
+            if (/claude code/i.test(body)) {
+              const claudeLabel = {
+                name: 'claude code',
+                color: '7c3aed',
+                description: 'Issues related to Claude Code usage'
+              };
+
+              try {
+                await github.rest.issues.getLabel({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  name: claudeLabel.name
+                });
+              } catch (error) {
+                if (error.status === 404) {
+                  await github.rest.issues.createLabel({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    name: claudeLabel.name,
+                    color: claudeLabel.color,
+                    description: claudeLabel.description
+                  });
+                }
+              }
+
+              await github.rest.issues.addLabels({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                labels: [claudeLabel.name]
+              });
+            }

.github/workflows/label-mlops.yml

@@ -1,17 +0,0 @@
-name: Label ML Ops Team Issues
-
-on:
-  issues:
-    types:
-      - opened
-
-jobs:
-  add-mlops-label:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check if ML Ops Team is selected
-        uses: actions-ecosystem/action-add-labels@v1
-        if: contains(github.event.issue.body, '### Are you a ML Ops Team?') && contains(github.event.issue.body, 'Yes')
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          labels: "mlops user request"

.github/workflows/llm-translation-testing.yml

@@ -4,38 +4,41 @@ on:
   workflow_dispatch:
     inputs:
       release_candidate_tag:
-        description: 'Release candidate tag/version'
+        description: "Release candidate tag/version"
         required: true
         type: string
   push:
     tags:
-      - 'v*-rc*'  # Triggers on release candidate tags like v1.0.0-rc1
-  
+      - "v*-rc*" # Triggers on release candidate tags like v1.0.0-rc1
+
+permissions:
+  contents: read
+
 jobs:
   run-llm-translation-tests:
     runs-on: ubuntu-latest
     timeout-minutes: 90
-    
+
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
+          persist-credentials: false
           ref: ${{ github.event.inputs.release_candidate_tag || github.ref }}
-      
+
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
-          python-version: '3.11'
-      
+          python-version: "3.11"
+
       - name: Install Poetry
-        uses: snok/install-poetry@v1
-        with:
-          version: latest
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-      
-      - name: Cache Poetry dependencies
-        uses: actions/cache@v3
+        run: |
+          pip install 'poetry==2.3.2'
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project true
+
+      - name: Restore Poetry dependencies cache
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
         with:
           path: |
             ~/.cache/pypoetry
@@ -43,15 +46,15 @@ jobs:
           key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
           restore-keys: |
             ${{ runner.os }}-poetry-
-      
+
       - name: Install dependencies
         run: |
           poetry install --with dev
-          poetry run pip install pytest-xdist pytest-timeout
-      
+          poetry run pip install 'pytest-xdist==3.8.0' 'pytest-timeout==2.4.0'
+
       - name: Create test results directory
         run: mkdir -p test-results
-      
+
       - name: Run LLM Translation Tests
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -61,13 +64,14 @@ jobs:
           AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
           AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
           AZURE_API_VERSION: ${{ secrets.AZURE_API_VERSION }}
-          # Add other API keys as needed
+          RC_TAG: ${{ github.event.inputs.release_candidate_tag || github.ref_name }}
+          COMMIT_SHA: ${{ github.sha }}
         run: |
           python .github/workflows/run_llm_translation_tests.py \
-            --tag "${{ github.event.inputs.release_candidate_tag || github.ref_name }}" \
-            --commit "${{ github.sha }}" \
+            --tag "$RC_TAG" \
+            --commit "$COMMIT_SHA" \
             || true  # Continue even if tests fail
-      
+
       - name: Display test summary
         if: always()
         run: |
@@ -79,9 +83,9 @@ jobs:
           else
             echo "Warning: Test report was not generated"
           fi
-      
+
       - name: Upload test artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         if: always()
         with:
           name: LLM-Translation-Artifact-${{ github.event.inputs.release_candidate_tag || github.ref_name }}

.github/workflows/load_test.yml

@@ -1,59 +0,0 @@
-name: Test Locust Load Test
-
-on:
-  workflow_run:
-    workflows: ["Build, Publish LiteLLM Docker Image. New Release"]
-    types:
-      - completed
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v1
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.x'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install PyGithub
-      - name: re-deploy proxy
-        run: |
-          echo "Current working directory: $PWD"
-          ls
-          python ".github/workflows/redeploy_proxy.py"
-        env:
-          LOAD_TEST_REDEPLOY_URL1: ${{ secrets.LOAD_TEST_REDEPLOY_URL1 }}
-          LOAD_TEST_REDEPLOY_URL2: ${{ secrets.LOAD_TEST_REDEPLOY_URL2 }}
-        working-directory: ${{ github.workspace }}
-      - name: Run Load Test
-        id: locust_run
-        uses: BerriAI/locust-github-action@master
-        with:
-          LOCUSTFILE: ".github/workflows/locustfile.py"
-          URL:  "https://post-release-load-test-proxy.onrender.com/"
-          USERS: "20"
-          RATE: "20"
-          RUNTIME: "300s"
-      - name: Process Load Test Stats
-        run: |
-          echo "Current working directory: $PWD"
-          ls
-          python ".github/workflows/interpret_load_test.py"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        working-directory: ${{ github.workspace }}
-      - name: Upload CSV as Asset to Latest Release
-        uses: xresloader/upload-to-github-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          file: "load_test_stats.csv;load_test.html"
-          update_latest_release: true
-          tag_name: "load-test"
-          overwrite: true

.github/workflows/locustfile.py

@@ -1,28 +0,0 @@
-from locust import HttpUser, task, between
-
-
-class MyUser(HttpUser):
-    wait_time = between(1, 5)
-
-    @task
-    def chat_completion(self):
-        headers = {
-            "Content-Type": "application/json",
-            "Authorization": "Bearer sk-8N1tLOOyH8TIxwOLahhIVg",
-            # Include any additional headers you may need for authentication, etc.
-        }
-
-        # Customize the payload with "model" and "messages" keys
-        payload = {
-            "model": "fake-openai-endpoint",
-            "messages": [
-                {"role": "system", "content": "You are a chat bot."},
-                {"role": "user", "content": "Hello, how are you?"},
-            ],
-            # Add more data as necessary
-        }
-
-        # Make a POST request to the "chat/completions" endpoint
-        response = self.client.post("chat/completions", json=payload, headers=headers)
-
-        # Print or log the response if needed

.github/workflows/main.yml

@@ -1,34 +0,0 @@
-name: Publish Dev Release to PyPI
-
-on:
-  workflow_dispatch:
-  
-jobs:
-  publish-dev-release:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8  # Adjust the Python version as needed
-
-      - name: Install dependencies
-        run: pip install toml twine
-
-      - name: Read version from pyproject.toml
-        id: read-version
-        run: |
-          version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
-          printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
-
-      - name: Check if version exists on PyPI
-        id: check-version
-        run: |
-          set -e
-          if twine check --repository-url https://pypi.org/simple/ "litellm==$LITELLM_VERSION" >/dev/null 2>&1; then
-            echo "Version $LITELLM_VERSION already exists on PyPI. Skipping publish."
-     

.github/workflows/publish-migrations.yml

@@ -1,206 +0,0 @@
-name: Publish Prisma Migrations
-
-permissions:
-  contents: write
-  pull-requests: write
-
-on:
-  push:
-    paths:
-      - 'schema.prisma'  # Check root schema.prisma
-    branches:
-      - main
-
-jobs:
-  publish-migrations:
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:14
-        env:
-          POSTGRES_DB: temp_db
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-        ports:
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-      
-      # Add shadow database service
-      postgres_shadow:
-        image: postgres:14
-        env:
-          POSTGRES_DB: shadow_db
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-        ports:
-          - 5433:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
-
-      - name: Install Dependencies
-        run: |
-          pip install prisma
-          pip install python-dotenv
-
-      - name: Generate Initial Migration if None Exists
-        env:
-          DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
-          DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
-          SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
-        run: |
-          mkdir -p deploy/migrations
-          echo 'provider = "postgresql"' > deploy/migrations/migration_lock.toml
-          
-          if [ -z "$(ls -A deploy/migrations/2* 2>/dev/null)" ]; then
-            echo "No existing migrations found, creating baseline..."
-            VERSION=$(date +%Y%m%d%H%M%S)
-            mkdir -p deploy/migrations/${VERSION}_initial
-            
-            echo "Generating initial migration..."
-            # Save raw output for debugging
-            prisma migrate diff \
-              --from-empty \
-              --to-schema-datamodel schema.prisma \
-              --shadow-database-url "${SHADOW_DATABASE_URL}" \
-              --script > deploy/migrations/${VERSION}_initial/raw_migration.sql
-            
-            echo "Raw migration file content:"
-            cat deploy/migrations/${VERSION}_initial/raw_migration.sql
-            
-            echo "Cleaning migration file..."
-            # Clean the file
-            sed '/^Installing/d' deploy/migrations/${VERSION}_initial/raw_migration.sql > deploy/migrations/${VERSION}_initial/migration.sql
-            
-            # Verify the migration file
-            if [ ! -s deploy/migrations/${VERSION}_initial/migration.sql ]; then
-              echo "ERROR: Migration file is empty after cleaning"
-              echo "Original content was:"
-              cat deploy/migrations/${VERSION}_initial/raw_migration.sql
-              exit 1
-            fi
-            
-            echo "Final migration file content:"
-            cat deploy/migrations/${VERSION}_initial/migration.sql
-            
-            # Verify it starts with SQL
-            if ! head -n 1 deploy/migrations/${VERSION}_initial/migration.sql | grep -q "^--\|^CREATE\|^ALTER"; then
-              echo "ERROR: Migration file does not start with SQL command or comment"
-              echo "First line is:"
-              head -n 1 deploy/migrations/${VERSION}_initial/migration.sql
-              echo "Full content is:"
-              cat deploy/migrations/${VERSION}_initial/migration.sql
-              exit 1
-            fi
-            
-            echo "Initial migration generated at $(date -u)" > deploy/migrations/${VERSION}_initial/README.md
-          fi
-
-      - name: Compare and Generate Migration
-        if: success()
-        env:
-          DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
-          DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
-          SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
-        run: |
-          # Create temporary migration workspace
-          mkdir -p temp_migrations
-          
-          # Copy existing migrations (will not fail if directory is empty)
-          cp -r deploy/migrations/* temp_migrations/ 2>/dev/null || true
-          
-          VERSION=$(date +%Y%m%d%H%M%S)
-          
-          # Generate diff against existing migrations or empty state
-          prisma migrate diff \
-            --from-migrations temp_migrations \
-            --to-schema-datamodel schema.prisma \
-            --shadow-database-url "${SHADOW_DATABASE_URL}" \
-            --script > temp_migrations/migration_${VERSION}.sql
-          
-          # Check if there are actual changes
-          if [ -s temp_migrations/migration_${VERSION}.sql ]; then
-            echo "Changes detected, creating new migration"
-            mkdir -p deploy/migrations/${VERSION}_schema_update
-            mv temp_migrations/migration_${VERSION}.sql deploy/migrations/${VERSION}_schema_update/migration.sql
-            echo "Migration generated at $(date -u)" > deploy/migrations/${VERSION}_schema_update/README.md
-          else
-            echo "No schema changes detected"
-            exit 0
-          fi
-
-      - name: Verify Migration
-        if: success()
-        env:
-          DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
-          DIRECT_URL: "postgresql://postgres:postgres@localhost:5432/temp_db"
-          SHADOW_DATABASE_URL: "postgresql://postgres:postgres@localhost:5433/shadow_db"
-        run: |
-          # Create test database
-          psql "${SHADOW_DATABASE_URL}" -c 'CREATE DATABASE migration_test;'
-          
-          # Apply all migrations in order to verify
-          for migration in deploy/migrations/*/migration.sql; do
-            echo "Applying migration: $migration"
-            psql "${SHADOW_DATABASE_URL}" -f $migration
-          done
-
-      # Add this step before create-pull-request to debug permissions
-      - name: Check Token Permissions
-        run: |
-          echo "Checking token permissions..."
-          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-               -H "Accept: application/vnd.github.v3+json" \
-               https://api.github.com/repos/BerriAI/litellm/collaborators
-          
-          echo "\nChecking if token can create PRs..."
-          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-               -H "Accept: application/vnd.github.v3+json" \
-               https://api.github.com/repos/BerriAI/litellm
-
-      # Add this debug step before git push
-      - name: Debug Changed Files
-        run: |
-          echo "Files staged for commit:"
-          git diff --name-status --staged
-          
-          echo "\nAll changed files:"
-          git status
-
-      - name: Create Pull Request
-        if: success()
-        uses: peter-evans/create-pull-request@v5
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          commit-message: "chore: update prisma migrations"
-          title: "Update Prisma Migrations"
-          body: |
-            Auto-generated migration based on schema.prisma changes.
-            
-            Generated files:
-            - deploy/migrations/${VERSION}_schema_update/migration.sql
-            - deploy/migrations/${VERSION}_schema_update/README.md
-          branch: feat/prisma-migration-${{ env.VERSION }}
-          base: main
-          delete-branch: true
-
-      - name: Generate and Save Migrations
-        run: |
-          # Only add migration files
-          git add deploy/migrations/
-          git status  # Debug what's being committed
-          git commit -m "chore: update prisma migrations" 

.github/workflows/publish_to_pypi.yml

@@ -0,0 +1,136 @@
+name: Publish to PyPI
+
+on:
+  workflow_dispatch:
+
+jobs:
+  preflight-checks:
+    name: Preflight Checks
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+    # No environment — read-only checks, no approval needed
+    outputs:
+      needs_publish: ${{ steps.check-litellm.outputs.needs_publish }}
+      version: ${{ steps.check-litellm.outputs.version }}
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Check litellm version on PyPI
+        id: check-litellm
+        run: |
+          VERSION=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
+          echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "Checking if litellm $VERSION exists on PyPI..."
+
+          HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm/$VERSION/json")
+          if [ "$HTTP_STATUS" = "200" ]; then
+            echo "litellm $VERSION already exists on PyPI. Skipping publish."
+            echo "needs_publish=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "litellm $VERSION not found on PyPI. Publish needed."
+            echo "needs_publish=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Sanity check proxy-extras version
+        run: |
+          # Read pinned version from requirements.txt
+          REQ_VERSION=$(grep -oP 'litellm-proxy-extras==\K[0-9.]+' requirements.txt)
+          if [ -z "$REQ_VERSION" ]; then
+            echo "::error::Could not find litellm-proxy-extras version in requirements.txt"
+            exit 1
+          fi
+          echo "requirements.txt pins litellm-proxy-extras==$REQ_VERSION"
+
+          # Read pinned version from pyproject.toml dependency
+          PYPROJECT_VERSION=$(python3 -c "
+          import re
+          with open('pyproject.toml') as f:
+              content = f.read()
+          match = re.search(r'litellm-proxy-extras\s*=\s*\{version\s*=\s*\"([^\"]+)\"', content)
+          if match:
+              print(match.group(1).lstrip('^~>='))
+          else:
+              import sys
+              print('::error::Could not find litellm-proxy-extras dependency in pyproject.toml', file=sys.stderr)
+              sys.exit(1)
+          ")
+          echo "pyproject.toml pins litellm-proxy-extras version: $PYPROJECT_VERSION"
+
+          # Check that both pinned versions match
+          if [ "$REQ_VERSION" != "$PYPROJECT_VERSION" ]; then
+            echo "::error::Version mismatch: requirements.txt has $REQ_VERSION but pyproject.toml has $PYPROJECT_VERSION"
+            exit 1
+          fi
+
+          # Check that the pinned version exists on PyPI
+          echo "Checking if litellm-proxy-extras $REQ_VERSION exists on PyPI..."
+          HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "https://pypi.org/pypi/litellm-proxy-extras/$REQ_VERSION/json")
+          if [ "$HTTP_STATUS" != "200" ]; then
+            echo "::error::litellm-proxy-extras $REQ_VERSION is not published on PyPI yet. Publish it before releasing litellm."
+            exit 1
+          fi
+          echo "litellm-proxy-extras $REQ_VERSION exists on PyPI. Sanity check passed."
+
+  publish-litellm:
+    name: Publish litellm to PyPI
+    needs: preflight-checks
+    if: needs.preflight-checks.outputs.needs_publish == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      id-token: write
+      contents: read
+    environment: pypi-publish
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Copy model prices backup
+        run: cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
+
+      - name: Install build tools
+        run: python -m pip install --upgrade pip build==1.4.2
+
+      - name: Build package
+        run: |
+          rm -rf build dist
+          python -m build
+
+      - name: Verify build artifacts
+        env:
+          EXPECTED_VERSION: ${{ needs.preflight-checks.outputs.version }}
+        run: |
+          echo "Contents of dist/:"
+          ls -la dist/
+          # Ensure we have both sdist and wheel
+          ls dist/*.tar.gz
+          ls dist/*.whl
+          # Verify built version matches expected
+          ls dist/ | grep -q "litellm-${EXPECTED_VERSION}" || {
+            echo "::error::Built artifacts do not match expected version $EXPECTED_VERSION"
+            ls dist/
+            exit 1
+          }
+
+      - name: Validate package metadata
+        run: |
+          pip install twine==6.2.0
+          twine check dist/*
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0

.github/workflows/read_pyproject_version.yml

@@ -3,7 +3,10 @@ name: Read Version from pyproject.toml
 on:
   push:
     branches:
-      - main  # Change this to the default branch of your repository
+      - main # Change this to the default branch of your repository
+
+permissions:
+  contents: read
 
 jobs:
   read-version:
@@ -11,20 +14,14 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
-          python-version: 3.8  # Adjust the Python version as needed
-
-      - name: Install dependencies
-        run: pip install toml
+          persist-credentials: false
 
       - name: Read version from pyproject.toml
         id: read-version
         run: |
-          version=$(python -c 'import toml; print(toml.load("pyproject.toml")["tool"]["commitizen"]["version"])')
+          version=$(grep -m1 '^version' pyproject.toml | sed 's/version = "\(.*\)"/\1/')
           printf "LITELLM_VERSION=%s" "$version" >> $GITHUB_ENV
 
       - name: Display version

.github/workflows/redeploy_proxy.py

@@ -1,20 +0,0 @@
-"""
-
-redeploy_proxy.py
-"""
-
-import os
-import requests
-import time
-
-# send a get request to this endpoint
-deploy_hook1 = os.getenv("LOAD_TEST_REDEPLOY_URL1")
-response = requests.get(deploy_hook1, timeout=20)
-
-
-deploy_hook2 = os.getenv("LOAD_TEST_REDEPLOY_URL2")
-response = requests.get(deploy_hook2, timeout=20)
-
-print("SENT GET REQUESTS to re-deploy proxy")
-print("sleeeping.... for 60s")
-time.sleep(60)

.github/workflows/reset_stable.yml

@@ -1,39 +0,0 @@
-name: Reset litellm_stable branch
-
-on:
-  release:
-    types: [published, created]
-jobs:
-  update-stable-branch:
-    if: ${{ startsWith(github.event.release.tag_name, 'v') && !endsWith(github.event.release.tag_name, '-stable') }}
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Reset litellm_stable_release_branch branch to the release commit
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          # Configure Git user
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          
-          # Fetch all branches and tags
-          git fetch --all
-
-          # Check if the litellm_stable_release_branch branch exists
-          if git show-ref --verify --quiet refs/remotes/origin/litellm_stable_release_branch; then
-            echo "litellm_stable_release_branch branch exists."
-            git checkout litellm_stable_release_branch
-          else
-            echo "litellm_stable_release_branch branch does not exist. Creating it."
-            git checkout -b litellm_stable_release_branch
-          fi
-
-          # Reset litellm_stable_release_branch branch to the release commit
-          git reset --hard $GITHUB_SHA
-
-          # Push the updated litellm_stable_release_branch branch
-          git push origin litellm_stable_release_branch --force

.github/workflows/run_observatory_tests.yml

@@ -0,0 +1,229 @@
+name: Run Observatory Tests
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Docker image tag to test (e.g. v1.61.0.rc1)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash (defaults to HEAD of current branch)"
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      tag:
+        description: "Docker image tag to test"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash of the release"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
+
+jobs:
+  observatory-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Validate tag input
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
+            exit 1
+          fi
+
+      - name: Start LiteLLM container
+        env:
+          TAG: ${{ inputs.tag }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+          WORKSPACE: ${{ github.workspace }}
+        run: |
+          docker run -d \
+            --name litellm-rc \
+            -p 4000:4000 \
+            -v "${WORKSPACE}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
+            -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
+            -e AZURE_API_KEY="${AZURE_API_KEY}" \
+            -e AZURE_API_BASE="${AZURE_API_BASE}" \
+            "litellm/litellm:${TAG}" \
+            --config /app/config.yaml --port 4000
+
+      - name: Wait for LiteLLM health check
+        run: |
+          echo "Waiting for LiteLLM to be ready..."
+          for i in $(seq 1 30); do
+            if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
+              echo "LiteLLM is healthy"
+              exit 0
+            fi
+            echo "Attempt $i/30 - not ready yet, waiting 10s..."
+            sleep 10
+          done
+          echo "LiteLLM failed to start within 5 minutes"
+          docker logs litellm-rc
+          exit 1
+
+      - name: Start cloudflared tunnel
+        run: |
+          # Install cloudflared (pinned version + checksum)
+          curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
+          echo "afdfadd1ef552e66bffc35246fe30a9bd578356d2d386de95585ccfc432472b8  /usr/local/bin/cloudflared" | sha256sum -c -
+          chmod +x /usr/local/bin/cloudflared
+
+          # Start a quick tunnel (no account needed) and capture the URL
+          cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
+          CLOUDFLARED_PID=$!
+          echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
+
+          # Wait for tunnel URL to appear in logs
+          echo "Waiting for tunnel URL..."
+          for i in $(seq 1 30); do
+            TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
+            if [ -n "$TUNNEL_URL" ]; then
+              echo "Tunnel URL: $TUNNEL_URL"
+              echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Failed to get tunnel URL"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Verify tunnel connectivity
+        run: |
+          echo "Testing tunnel at ${TUNNEL_URL}..."
+          # Quick tunnels need time for DNS propagation; retry to avoid
+          # transient NXDOMAIN (curl exit code 6) on first attempt.
+          for i in $(seq 1 10); do
+            if curl -sf "${TUNNEL_URL}/health/liveliness" > /dev/null 2>&1; then
+              echo "Tunnel is working (attempt $i)"
+              exit 0
+            fi
+            echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
+            sleep 5
+          done
+          echo "Tunnel failed to become reachable after 50s"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Trigger observatory test run
+        id: trigger
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+        run: |
+          PAYLOAD=$(jq -n \
+            --arg url "${TUNNEL_URL}" \
+            --arg key "${LITELLM_MASTER_KEY}" \
+            '{
+              deployment_url: $url,
+              api_key: $key,
+              test_suite: "TestOAIAzureRelease",
+              models: ["gpt-4o-mini", "gpt-4o"]
+            }')
+          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
+            -H "Content-Type: application/json" \
+            -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
+            -d "$PAYLOAD")
+          HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+          BODY=$(echo "$RESPONSE" | head -n -1)
+          echo "Response ($HTTP_CODE): $BODY"
+          if [ "$HTTP_CODE" -ge 400 ]; then
+            echo "Failed to trigger test run"
+            exit 1
+          fi
+
+          # Extract request_id for polling this specific run
+          REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
+          if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
+            echo "Failed to extract request_id from response"
+            exit 1
+          fi
+          echo "Request ID: $REQUEST_ID"
+          echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
+
+      - name: Poll for test completion
+        id: poll
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+          REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
+        run: |
+          TIMEOUT=900  # 15 minutes
+          INTERVAL=30
+          ELAPSED=0
+          while [ $ELAPSED -lt $TIMEOUT ]; do
+            STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
+              -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
+            RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
+            echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
+
+            if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
+              echo "Test finished with status: $RUN_STATUS"
+              echo "$STATUS" > /tmp/observatory_result.json
+              exit 0
+            fi
+
+            sleep $INTERVAL
+            ELAPSED=$((ELAPSED + INTERVAL))
+          done
+          echo "Timed out waiting for test to complete after ${TIMEOUT}s"
+          exit 1
+
+      - name: Verify test results
+        run: |
+          RESULT=$(cat /tmp/observatory_result.json)
+          echo "Full result: $RESULT"
+
+          STATUS=$(echo "$RESULT" | jq -r '.status')
+          TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
+          FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
+          ERROR=$(echo "$RESULT" | jq -r '.error // empty')
+
+          echo "Status: $STATUS"
+          echo "Test passed: $TEST_PASSED"
+          echo "Failure rate: $FAILURE_RATE"
+
+          if [ -n "$ERROR" ]; then
+            echo "Error: $ERROR"
+          fi
+
+          if [ "$STATUS" = "failed" ]; then
+            echo "Test run failed"
+            exit 1
+          fi
+
+          if [ "$TEST_PASSED" != "true" ]; then
+            echo "Tests did not pass (failure rate: $FAILURE_RATE)"
+            exit 1
+          fi
+
+          echo "All tests passed!"
+
+      - name: Print LiteLLM logs on failure
+        if: failure()
+        run: |
+          docker logs litellm-rc 2>/dev/null || true
+          cat /tmp/cloudflared.log 2>/dev/null || true
+
+      - name: Cleanup
+        if: always()
+        run: |
+          kill "$CLOUDFLARED_PID" 2>/dev/null || true
+          docker rm -f litellm-rc 2>/dev/null || true

.github/workflows/scan_duplicate_issues.yml

@@ -0,0 +1,48 @@
+name: Scan Duplicate Issues (One-Time)
+
+on:
+  workflow_dispatch:
+    inputs:
+      threshold:
+        description: "Similarity threshold (0-1)"
+        required: false
+        default: "0.85"
+      close:
+        description: "Actually close duplicates (false = dry run)"
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout scripts
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          sparse-checkout: .github/scripts
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.13"
+
+      - name: Scan for duplicate issues
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INPUT_THRESHOLD: ${{ inputs.threshold }}
+          INPUT_CLOSE: ${{ inputs.close }}
+        run: |
+          CLOSE_FLAG=""
+          if [ "$INPUT_CLOSE" = "true" ]; then
+            CLOSE_FLAG="--close"
+          fi
+          python3 .github/scripts/close_duplicate_issues.py \
+            --scan \
+            --repo ${{ github.repository }} \
+            --threshold "$INPUT_THRESHOLD" \
+            $CLOSE_FLAG

.github/workflows/scorecard.yml

@@ -0,0 +1,47 @@
+name: Scorecard supply-chain security
+
+on:
+  branch_protection_rule:
+  schedule:
+    - cron: '27 12 * * 4'
+  push:
+    branches: ["main"]
+
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    if: github.event.repository.default_branch == github.ref_name
+    permissions:
+      security-events: write
+      id-token: write
+      # Uncomment for private repos if needed:
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: Run analysis
+        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          publish_results: true
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      - name: Upload to code scanning
+        uses: github/codeql-action/upload-sarif@c10b806170c8ee63ea24152429041b5624f0baf5 # v4.35.1
+        with:
+          sarif_file: results.sarif

.github/workflows/simple_pypi_publish.yml

@@ -1,67 +0,0 @@
-name: Simple PyPI Publish
-
-on:
-  workflow_dispatch:
-    inputs:
-      version:
-        description: 'Version to publish (e.g., 1.74.10)'
-        required: true
-        type: string
-
-env:
-  TWINE_USERNAME: __token__
-
-jobs:
-  publish:
-    runs-on: ubuntu-latest
-    if: github.repository == 'BerriAI/litellm'
-    
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.8'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install toml build wheel twine
-
-      - name: Update version in pyproject.toml
-        run: |
-          python -c "
-          import toml
-          
-          with open('pyproject.toml', 'r') as f:
-              data = toml.load(f)
-          
-          data['tool']['poetry']['version'] = '${{ github.event.inputs.version }}'
-          
-          with open('pyproject.toml', 'w') as f:
-              toml.dump(data, f)
-          
-          print(f'Updated version to ${{ github.event.inputs.version }}')
-          "
-
-      - name: Copy model prices file
-        run: |
-          cp model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
-
-      - name: Build package
-        run: |
-          rm -rf build dist
-          python -m build
-
-      - name: Publish to PyPI
-        env:
-          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
-        run: |
-          twine upload dist/*
-
-      - name: Output success
-        run: |
-          echo "✅ Successfully published litellm v${{ github.event.inputs.version }} to PyPI"
-          echo "📦 Package: https://pypi.org/project/litellm/${{ github.event.inputs.version }}/" 

.github/workflows/stale.yml

@@ -2,19 +2,24 @@ name: "Stale Issue Management"
 
 on:
   schedule:
-    - cron: '0 0 * * *' # Runs daily at midnight UTC
+    - cron: "0 0 * * *" # Runs daily at midnight UTC
   workflow_dispatch:
 
+permissions:
+  issues: write
+  pull-requests: write
+
 jobs:
   stale:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@1160a2240286f5da8ec72b1c0816ce2481aabf84 # v8
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           stale-issue-message: "This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
           stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs."
-          days-before-stale: 90        # Revert to 60 days
-          days-before-close: 7         # Revert to 7 days
+          days-before-stale: 90 # Revert to 60 days
+          days-before-close: 7 # Revert to 7 days
           stale-issue-label: "stale"
-          operations-per-run: 1000
+          operations-per-run: 1000

.github/workflows/sync-schema.yml

@@ -0,0 +1,73 @@
+name: Sync schema.prisma copies
+
+on:
+  pull_request:
+    paths:
+      - 'schema.prisma'
+
+# Scoped to ONLY the permissions needed:
+# - contents:write to push the sync commit to the PR branch
+# - pull-requests:read is implicit (needed to check out the PR)
+permissions:
+  contents: write
+
+jobs:
+  sync:
+    name: Copy root schema to proxy and proxy-extras
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    # Only run on PRs from branches in THIS repo (not forks).
+    # Fork PRs cannot push back to the head branch with GITHUB_TOKEN,
+    # and pull_request events from forks have read-only tokens anyway.
+    # Also reject PRs from branches named after protected branches to
+    # prevent pushing directly to main/master.
+    if: >-
+      github.event.pull_request.head.repo.full_name == github.repository
+      && github.head_ref != 'main'
+      && github.head_ref != 'master'
+    steps:
+      - name: Checkout PR branch by SHA
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          # Use the merge commit SHA for safety — github.head_ref is an
+          # attacker-controlled string (the branch name) and could contain
+          # unusual characters that cause unexpected git behavior.
+          ref: ${{ github.event.pull_request.head.sha }}
+          persist-credentials: true  # needed for git push
+
+      - name: Reject symlinked schema files
+        run: |
+          for f in schema.prisma litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; do
+            if [ -L "$f" ]; then
+              echo "::error file=$f::$f is a symlink, which is not allowed"
+              exit 1
+            fi
+          done
+
+      - name: Copy root schema to other locations
+        run: |
+          cp schema.prisma litellm/proxy/schema.prisma
+          cp schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+
+      - name: Check for changes
+        id: diff
+        run: |
+          if git diff --quiet -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+            echo "Schemas already in sync. Nothing to do."
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            echo "Schema copies need updating."
+          fi
+
+      - name: Commit synced schemas
+        if: steps.diff.outputs.changed == 'true'
+        run: |
+          # Push to the PR's head branch (need the branch name for git push).
+          # We checked out by SHA above for safety, so configure the push target explicitly.
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -B "$GITHUB_HEAD_REF"
+          git add -- litellm/proxy/schema.prisma litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+          git commit -m "chore: sync schema.prisma copies from root"
+          git push origin "HEAD:$GITHUB_HEAD_REF"

.github/workflows/test-linting.yml

@@ -2,7 +2,10 @@ name: LiteLLM Linting
 
 on:
   pull_request:
-    branches: [ main ]
+    branches: [main]
+
+permissions:
+  contents: read
 
 jobs:
   lint:
@@ -10,49 +13,103 @@ jobs:
     timeout-minutes: 5
 
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 0
+          clean: true
+          persist-credentials: false
 
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.12'
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
 
-    - name: Install Poetry
-      uses: snok/install-poetry@v1
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
 
-    - name: Install dependencies
-      run: |
-        poetry install --with dev
-        poetry run pip install openai==1.100.1
+      - name: Clean Python cache
+        run: |
+          find . -type d -name "__pycache__" -exec rm -rf {} + || true
+          find . -name "*.pyc" -delete || true
 
-    - name: Run Black formatting
-      run: |
-        cd litellm
-        poetry run black .
-        cd ..
+      - name: Check poetry.lock is up to date
+        run: |
+          poetry check --lock || (echo "❌ poetry.lock is out of sync with pyproject.toml. Run 'poetry lock' locally and commit the result." && exit 1)
 
-    - name: Run Ruff linting
-      run: |
-        cd litellm
-        poetry run ruff check .
-        cd ..
+      - name: Install dependencies
+        run: |
+          poetry install --with dev
 
-    - name: Print OpenAI version
-      run: |
-        poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
+      - name: Check Black formatting
+        run: |
+          cd litellm
+          poetry run black --check --exclude '/enterprise/' .
+          cd ..
 
-    - name: Run MyPy type checking
-      run: |
-        cd litellm
-        poetry run mypy . --ignore-missing-imports
-        cd ..
+      - name: Debug - Check file state
+        run: |
+          echo "Current branch:"
+          git branch --show-current
+          echo "Last 3 commits:"
+          git log --oneline -3
+          echo "File content around line 43:"
+          head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
 
-    - name: Check for circular imports
-      run: |
-        cd litellm
-        poetry run python ../tests/documentation_tests/test_circular_imports.py
-        cd ..
+      - name: Run Ruff linting
+        run: |
+          cd litellm
+          poetry run ruff check .
+          cd ..
 
-    - name: Check import safety
-      run: |
-        poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+      - name: Print OpenAI version
+        run: |
+          poetry run python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
+
+      - name: Run MyPy type checking
+        run: |
+          cd litellm
+          poetry run mypy . 
+          cd ..
+
+      - name: Check for circular imports
+        run: |
+          cd litellm
+          poetry run python ../tests/documentation_tests/test_circular_imports.py
+          cd ..
+
+      - name: Check import safety
+        run: |
+          poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+  secret-scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Run secret scan test
+        run: |
+          pip install 'pytest==9.0.2'
+          pytest tests/litellm/test_no_hardcoded_secrets.py -v
+
+      - name: Run ggshield secret scan
+        env:
+          GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
+        run: |
+          if [ -n "$GITGUARDIAN_API_KEY" ]; then
+            pip install 'ggshield==1.48.0'
+            ggshield secret scan repo .
+          else
+            echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
+          fi

.github/workflows/test-litellm-matrix.yml

@@ -0,0 +1,174 @@
+name: LiteLLM Unit Tests (Matrix)
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+# Cancel in-progress runs for the same PR
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20 # Increased from 15 to 20
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group:
+          # tests/test_litellm split by subdirectory (~560 files total)
+          # Vertex AI tests separated for better isolation (prevent auth/env pollution)
+          - name: "llms-vertex"
+            path: "tests/test_litellm/llms/vertex_ai"
+            workers: 1
+            reruns: 2
+          - name: "llms-other"
+            path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+            workers: 2
+            reruns: 2
+          # tests/test_litellm/proxy split by subdirectory (~180 files total)
+          - name: "proxy-guardrails"
+            path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
+            workers: 2
+            reruns: 2
+          - name: "proxy-core"
+            path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
+            workers: 2
+            reruns: 2
+          - name: "proxy-misc"
+            path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
+            workers: 2
+            reruns: 2
+          - name: "integrations"
+            path: "tests/test_litellm/integrations"
+            workers: 2
+            reruns: 3 # Integration tests tend to be flakier
+          - name: "core-utils"
+            path: "tests/test_litellm/litellm_core_utils"
+            workers: 2
+            reruns: 1
+          - name: "other-1"
+            # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
+            path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
+            workers: 2
+            reruns: 2
+          - name: "other-2"
+            # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
+            path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
+            workers: 2
+            reruns: 2
+          - name: "other-3"
+            # remaining dirs ≈ 8.0k lines
+            path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
+            workers: 2
+            reruns: 2
+          - name: "root"
+            path: "tests/test_litellm/test_*.py"
+            workers: 2
+            reruns: 2
+          # tests/proxy_unit_tests split alphabetically (~48 files total)
+          - name: "proxy-unit-a1"
+            # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
+            path: "tests/proxy_unit_tests/test_[a-j]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-a2"
+            # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
+            path: "tests/proxy_unit_tests/test_[k-o]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b1"
+            # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
+            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b2"
+            # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
+            path: "tests/proxy_unit_tests/test_proxy_server.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b3"
+            # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
+            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b4"
+            # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
+            path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b5"
+            # proxy_token_counter (1279) - runs independently from utils
+            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b6"
+            # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
+            path: "tests/proxy_unit_tests/test_[r-t]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b7"
+            # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
+            path: "tests/proxy_unit_tests/test_[u-z]*.py"
+            workers: 2
+            reruns: 1
+
+    name: test (${{ matrix.test-group.name }})
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
+          poetry run pip install google-genai==1.22.0 \
+            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          poetry run pip install nodejs-wheel-binaries==24.13.1
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run tests - ${{ matrix.test-group.name }}
+        run: |
+          poetry run pytest ${{ matrix.test-group.path }} \
+            --tb=short -vv \
+            --maxfail=10 \
+            -n ${{ matrix.test-group.workers }} \
+            --reruns ${{ matrix.test-group.reruns }} \
+            --reruns-delay 1 \
+            --dist=loadscope \
+            --durations=20

.github/workflows/test-litellm-ui-build.yml

@@ -0,0 +1,34 @@
+name: UI Build Check
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  build-ui:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: ui/litellm-dashboard
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: ui/litellm-dashboard/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build
+        run: npm run build

.github/workflows/test-litellm.yml

@@ -1,8 +1,15 @@
 name: LiteLLM Mock Tests (folder - tests/test_litellm)
 
+# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs
+# the same tests in parallel across 10 jobs for faster CI times.
+# Kept for manual debugging only.
 on:
-  pull_request:
-    branches: [ main ]
+  workflow_dispatch: # Manual trigger only
+  # pull_request:
+  #   branches: [ main ]
+
+permissions:
+  contents: read
 
 jobs:
   test:
@@ -10,34 +17,37 @@ jobs:
     timeout-minutes: 25
 
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
 
-    - name: Thank You Message
-      run: |
-        echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
-        echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
+      - name: Thank You Message
+        run: |
+          echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
+          echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
 
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.12'
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
 
-    - name: Install Poetry
-      uses: snok/install-poetry@v1
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
 
-    - name: Install dependencies
-      run: |
-        poetry install --with dev,proxy-dev --extras "proxy semantic-router"
-        poetry run pip install "pytest-retry==1.6.3"
-        poetry run pip install pytest-xdist
-        poetry run pip install "google-genai==1.22.0"
-        poetry run pip install "google-cloud-aiplatform>=1.38"
-        poetry run pip install "fastapi-offline==1.7.3"
-    - name: Setup litellm-enterprise as local package
-      run: |
-        cd enterprise
-        python -m pip install -e .
-        cd ..
-    - name: Run tests
-      run: |
-        poetry run pytest tests/test_litellm -x -vv -n 4 
+      - name: Install dependencies
+        run: |
+          poetry lock
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          poetry run pip install "pytest-retry==1.6.3"
+          poetry run pip install 'pytest-xdist==3.8.0'
+          poetry run pip install "google-genai==1.22.0"
+          poetry run pip install "google-cloud-aiplatform==1.115.0"
+          poetry run pip install "fastapi-offline==1.7.3"
+          poetry run pip install "python-multipart==0.0.22"
+          poetry run pip install "openapi-core==0.23.0"
+      - name: Setup litellm-enterprise as local package
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+      - name: Run tests
+        run: |
+          poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

.github/workflows/test-mcp.yml

@@ -0,0 +1,52 @@
+name: LiteLLM MCP Tests (folder - tests/mcp_tests)
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Thank You Message
+        run: |
+          echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
+          echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
+
+      - name: Install dependencies
+        run: |
+          poetry lock
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          poetry run pip install "pytest==7.3.1"
+          poetry run pip install "pytest-retry==1.6.3"
+          poetry run pip install "pytest-cov==5.0.0"
+          poetry run pip install "pytest-asyncio==0.21.1"
+          poetry run pip install "respx==0.22.0"
+          poetry run pip install "pydantic==2.11.0"
+          poetry run pip install "mcp==1.25.0"
+          poetry run pip install 'pytest-xdist==3.8.0'
+
+      - name: Setup litellm-enterprise as local package
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Run MCP tests
+        run: |
+          poetry run pytest tests/mcp_tests -x -vv -n 4 --cov=litellm --cov-report=xml --durations=5

.github/workflows/test-model-map.yaml

@@ -0,0 +1,20 @@
+name: Validate model_prices_and_context_window.json
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  validate-model-prices-json:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Validate model_prices_and_context_window.json
+        run: |
+          jq empty model_prices_and_context_window.json

.github/workflows/test-proxy-e2e-azure-batches.yml

@@ -0,0 +1,97 @@
+name: Proxy E2E Azure Batches Tests
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  proxy_e2e_azure_batches_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.0.0
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-e2e-batches-
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy"
+          poetry run pip install psycopg2-binary==2.9.11 uvicorn==0.42.0 fastapi==0.135.2 httpx==0.28.1 tenacity==9.1.4
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          poetry run pip install nodejs-wheel-binaries==24.13.1
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run Prisma migrations
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+        run: |
+          cd litellm/proxy
+          poetry run prisma migrate deploy --schema schema.prisma
+          cd ../..
+
+      - name: Run Azure Batch E2E Tests
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+          USE_LOCAL_LITELLM: "true"
+          USE_MOCK_MODELS: "true"
+          USE_STATE_TRACKER: "true"
+          LITELLM_LOG: DEBUG
+        run: |
+          poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
+            -vv -s -k "test_e2e_managed_batch" \
+            --tb=short \
+            --maxfail=3 \
+            --durations=10

.github/workflows/test-unit-caching-redis.yml

@@ -0,0 +1,38 @@
+name: "Unit Tests: Caching (Redis)"
+
+# Uses cloud Redis credentials — only runs on trusted branches, not PRs.
+# This prevents external PRs from accessing Redis credentials.
+on:
+  push:
+    branches: [main, "litellm_*"]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  caching-redis:
+    uses: ./.github/workflows/_test-unit-services-base.yml
+    with:
+      # Redis-only tests that do NOT require provider API keys.
+      # Tests needing API keys (test_caching.py, test_caching_ssl.py, test_prometheus_service.py,
+      # test_router_caching.py) are in Phase 3 integration workflows.
+      test-path: >-
+        tests/local_testing/test_dual_cache.py
+        tests/local_testing/test_redis_batch_optimizations.py
+        tests/local_testing/test_router_utils.py
+      workers: 2
+      reruns: 2
+      timeout-minutes: 20
+      enable-redis: true
+      enable-postgres: false
+    secrets:
+      REDIS_HOST: ${{ secrets.REDIS_HOST }}
+      REDIS_PORT: ${{ secrets.REDIS_PORT }}
+      REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }}
+      DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test-unit-core-utils.yml

@@ -0,0 +1,20 @@
+name: "Unit Tests: Core Utilities"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  core-utils:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/litellm_core_utils"
+      workers: 2
+      reruns: 1

.github/workflows/test-unit-documentation.yml

@@ -0,0 +1,67 @@
+name: "Unit Tests: Documentation Validation"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  documentation:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          poetry run pip install google-genai==1.22.0 \
+            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          poetry run pip install nodejs-wheel-binaries==24.13.1
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      # Run the same documentation tests that CircleCI ran (as direct Python scripts)
+      - name: Run documentation validation tests
+        run: |
+          poetry run python ./tests/documentation_tests/test_env_keys.py
+          poetry run python ./tests/documentation_tests/test_router_settings.py
+          poetry run python ./tests/documentation_tests/test_api_docs.py
+          poetry run python ./tests/documentation_tests/test_circular_imports.py

.github/workflows/test-unit-enterprise-routing.yml

@@ -0,0 +1,24 @@
+name: "Unit Tests: Enterprise, Google GenAI & Routing"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  enterprise-routing:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: >-
+        tests/test_litellm/enterprise
+        tests/test_litellm/google_genai
+        tests/test_litellm/router_utils
+        tests/test_litellm/router_strategy
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-integrations.yml

@@ -0,0 +1,20 @@
+name: "Unit Tests: Integrations (Callbacks & Logging)"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  integrations:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/integrations"
+      workers: 2
+      reruns: 3

.github/workflows/test-unit-llm-providers.yml

@@ -0,0 +1,29 @@
+name: "Unit Tests: LLM Provider Transformations"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  vertex-ai:
+    name: Vertex AI
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/llms/vertex_ai"
+      workers: 1
+      reruns: 2
+
+  other-providers:
+    name: All Other Providers
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-misc.yml

@@ -0,0 +1,31 @@
+name: "Unit Tests: MCP, Secrets, Containers & Misc"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  misc:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: >-
+        tests/test_litellm/secret_managers
+        tests/test_litellm/a2a_protocol
+        tests/test_litellm/anthropic_interface
+        tests/test_litellm/completion_extras
+        tests/test_litellm/containers
+        tests/test_litellm/experimental_mcp_client
+        tests/test_litellm/images
+        tests/test_litellm/interactions
+        tests/test_litellm/passthrough
+        tests/test_litellm/vector_stores
+        tests/test_litellm/test_*.py
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-proxy-auth.yml

@@ -0,0 +1,20 @@
+name: "Unit Tests: Proxy Auth & Key Management"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy-auth:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client"
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-proxy-db.yml

@@ -0,0 +1,45 @@
+name: "Unit Tests: Proxy DB Operations"
+
+# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
+on:
+  push:
+    branches: [main, "litellm_*"]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy-db:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Key generation tests must NOT run in parallel (event loop conflicts with logging worker)
+          - test-group: key-generation
+            test-path: "tests/proxy_unit_tests/test_key_generate_prisma.py"
+            workers: 0
+            timeout: 30
+          - test-group: auth-checks
+            test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py"
+            workers: 8
+            timeout: 20
+          - test-group: remaining
+            test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py"
+            workers: 8
+            timeout: 20
+    uses: ./.github/workflows/_test-unit-services-base.yml
+    with:
+      test-path: ${{ matrix.test-path }}
+      workers: ${{ matrix.workers }}
+      reruns: 2
+      timeout-minutes: ${{ matrix.timeout }}
+      enable-redis: false
+      enable-postgres: true
+    secrets:
+      DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test-unit-proxy-endpoints.yml

@@ -0,0 +1,35 @@
+name: "Unit Tests: Proxy API Endpoints"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy-endpoints:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: >-
+        tests/test_litellm/proxy/management_endpoints
+        tests/test_litellm/proxy/guardrails
+        tests/test_litellm/proxy/management_helpers
+        tests/test_litellm/proxy/anthropic_endpoints
+        tests/test_litellm/proxy/google_endpoints
+        tests/test_litellm/proxy/openai_files_endpoint
+        tests/test_litellm/proxy/response_api_endpoints
+        tests/test_litellm/proxy/image_endpoints
+        tests/test_litellm/proxy/vector_store_endpoints
+        tests/test_litellm/proxy/agent_endpoints
+        tests/test_litellm/proxy/discovery_endpoints
+        tests/test_litellm/proxy/health_endpoints
+        tests/test_litellm/proxy/public_endpoints
+        tests/test_litellm/proxy/prompts
+        tests/test_litellm/proxy/ui_crud_endpoints
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-proxy-infra.yml

@@ -0,0 +1,28 @@
+name: "Unit Tests: Proxy Infrastructure"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy-infra:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: >-
+        tests/test_litellm/proxy/db
+        tests/test_litellm/proxy/middleware
+        tests/test_litellm/proxy/spend_tracking
+        tests/test_litellm/proxy/pass_through_endpoints
+        tests/test_litellm/proxy/_experimental
+        tests/test_litellm/proxy/experimental
+        tests/test_litellm/proxy/common_utils
+        tests/test_litellm/proxy/test_*.py
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-proxy-legacy.yml

@@ -0,0 +1,96 @@
+name: "Unit Tests: Proxy Legacy Tests"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group:
+          - name: "auth-and-jwt"
+            path: "tests/proxy_unit_tests/test_[a-j]*.py"
+          - name: "key-generation"
+            path: "tests/proxy_unit_tests/test_[k-o]*.py"
+          - name: "proxy-config"
+            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+          - name: "proxy-server"
+            path: "tests/proxy_unit_tests/test_proxy_server.py"
+          - name: "proxy-server-extras"
+            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
+          - name: "proxy-utils"
+            path: "tests/proxy_unit_tests/test_proxy_utils.py"
+          - name: "proxy-token-counter"
+            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
+          - name: "proxy-response-and-misc"
+            path: "tests/proxy_unit_tests/test_[r-t]*.py"
+          - name: "proxy-user-auth-and-spend"
+            path: "tests/proxy_unit_tests/test_[u-z]*.py"
+
+    name: ${{ matrix.test-group.name }}
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        run: pip install 'poetry==2.3.2'
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          poetry run pip install google-genai==1.22.0 \
+            google-cloud-aiplatform==1.115.0 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core==0.23.0
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          poetry run pip install nodejs-wheel-binaries==24.13.1
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run tests - ${{ matrix.test-group.name }}
+        env:
+          TEST_PATH: ${{ matrix.test-group.path }}
+        run: |
+          poetry run pytest ${TEST_PATH} \
+            --tb=short -vv \
+            --maxfail=10 \
+            -n 2 \
+            --reruns 1 \
+            --reruns-delay 1 \
+            --dist=loadscope \
+            --durations=20

.github/workflows/test-unit-responses-caching-types.yml

@@ -0,0 +1,20 @@
+name: "Unit Tests: Responses, Caching & Types"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  responses-caching-types:
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
+      workers: 2
+      reruns: 2

.github/workflows/test-unit-security.yml

@@ -0,0 +1,28 @@
+name: "Unit Tests: Security"
+
+# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
+on:
+  push:
+    branches: [main, "litellm_*"]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  security:
+    uses: ./.github/workflows/_test-unit-services-base.yml
+    with:
+      test-path: "tests/proxy_security_tests/"
+      workers: 1
+      reruns: 2
+      timeout-minutes: 20
+      enable-redis: false
+      enable-postgres: true
+    secrets:
+      DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}

.github/workflows/test_server_root_path.yml

@@ -0,0 +1,98 @@
+name: Test Proxy SERVER_ROOT_PATH Routing
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  test-server-root-path:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    strategy:
+      matrix:
+        root_path: ["/api/v1", "/llmproxy"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12
+
+      - name: Build Docker image
+        uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 #v6.14
+        with:
+          context: .
+          file: ./docker/Dockerfile.non_root
+          tags: litellm-test:${{ github.sha }}
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Start LiteLLM container with SERVER_ROOT_PATH
+        run: |
+          docker run -d \
+            --name litellm-test \
+            -p 4000:4000 \
+            -e SERVER_ROOT_PATH="${{ matrix.root_path }}" \
+            -e LITELLM_MASTER_KEY="sk-1234" \
+            litellm-test:${{ github.sha }} \
+            --detailed_debug
+
+      - name: Wait for container to be healthy
+        run: |
+          echo "Waiting for LiteLLM to start..."
+          max_attempts=30
+          attempt=0
+
+          while [ $attempt -lt $max_attempts ]; do
+            if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then
+              echo "LiteLLM started successfully"
+              break
+            fi
+            attempt=$((attempt + 1))
+            echo "Attempt $attempt/$max_attempts - waiting for server to start..."
+            sleep 2
+          done
+
+          if [ $attempt -eq $max_attempts ]; then
+            echo "Server failed to start within timeout"
+            docker logs litellm-test
+            exit 1
+          fi
+
+          sleep 5
+
+      - name: Show container logs
+        if: always()
+        run: docker logs litellm-test
+
+      - name: Test UI endpoint with root path
+        run: |
+          ROOT_PATH="${{ matrix.root_path }}"
+          echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/"
+
+          for i in 1 2 3; do
+            content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/")
+            if echo "$content" | grep -q -E "(html|<!DOCTYPE|<head|<body)"; then
+              echo "UI page contains valid HTML content"
+              exit 0
+            fi
+            echo "Attempt $i/3 - no valid HTML, retrying in 5s..."
+            sleep 5
+          done
+          echo "UI page does not contain expected HTML content"
+          echo "Response: $content"
+          docker logs litellm-test
+          exit 1
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker stop litellm-test || true
+          docker rm litellm-test || true

.github/workflows/zizmor.yml

@@ -0,0 +1,31 @@
+name: GitHub Actions Security Analysis
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  zizmor:
+    name: zizmor
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      security-events: write
+      contents: read
+      actions: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Run zizmor
+        uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2

.gitignore

@@ -1,6 +1,8 @@
 .python-version
 .venv
+.venv_policy_test
 .env
+.claude
 .newenv
 newenv/*
 litellm/proxy/myenv/*
@@ -59,9 +61,6 @@ litellm/proxy/_super_secret_config.yaml
 litellm/proxy/myenv/bin/activate
 litellm/proxy/myenv/bin/Activate.ps1
 myenv/*
-litellm/proxy/_experimental/out/404/index.html
-litellm/proxy/_experimental/out/model_hub/index.html
-litellm/proxy/_experimental/out/onboarding/index.html
 litellm/tests/log.txt
 litellm/tests/langfuse.log
 litellm/tests/langfuse.log
@@ -74,9 +73,6 @@ tests/local_testing/log.txt
 litellm/proxy/_new_new_secret_config.yaml
 litellm/proxy/custom_guardrail.py
 .mypy_cache/*
-litellm/proxy/_experimental/out/404.html
-litellm/proxy/_experimental/out/404.html
-litellm/proxy/_experimental/out/model_hub.html
 .mypy_cache/*
 litellm/proxy/application.log
 tests/llm_translation/vertex_test_account.json
@@ -93,7 +89,16 @@ tests/test_custom_dir/*
 test.py
 
 litellm_config.yaml
+!.github/observatory/litellm_config.yaml
 .cursor
 .vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*
 update_model_cost_map.py
+tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
+scripts/test_vertex_ai_search.py
+LAZY_LOADING_IMPROVEMENTS.md
+STABILIZATION_TODO.md
+**/test-results
+**/playwright-report
+**/*.storageState.json
+**/coverage

.npmrc

@@ -0,0 +1,5 @@
+# Supply-chain hardening
+# Packages needing lifecycle scripts: npm rebuild <pkg>
+ignore-scripts=true
+# Protects local npm install only — npm ci (used in CI) ignores this
+min-release-age=3d

.pre-commit-config.yaml

@@ -14,12 +14,12 @@ repos:
         types: [python]
         files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
         exclude: ^litellm/__init__.py$
-    # -   id: black
-    #     name: black
-    #     entry: poetry run black
-    #     language: system
-    #     types: [python]
-    #     files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
+    -   id: black
+        name: black
+        entry: poetry run black
+        language: system
+        types: [python]
+        files: (litellm/|litellm_proxy_extras/).*\.py
 -   repo: https://github.com/pycqa/flake8
     rev: 7.0.0  # The version of flake8 to use
     hooks:

.semgrep/rules/README.md

@@ -0,0 +1,22 @@
+# Custom Semgrep rules for LiteLLM
+
+Add custom rule YAML files here. Semgrep loads all `.yml`/`.yaml` files under this directory.
+
+**Run only custom rules (CI / fail on findings):**
+
+```bash
+semgrep scan --config .semgrep/rules . --error
+```
+
+**Run with registry + custom rules:**
+
+```bash
+semgrep scan --config auto --config .semgrep/rules .
+```
+
+**Layout:**
+
+- `python/` – Python-specific rules (security, patterns)
+- Add more subdirs as needed (e.g. `generic/` for language-agnostic rules)
+
+See [Semgrep rule syntax](https://semgrep.dev/docs/writing-rules/rule-syntax/).

.semgrep/rules/python/reliability/unbounded-memory.yml

@@ -0,0 +1,17 @@
+# Unbounded memory growth – data structures without a clear max limit
+# Can lead to OOM under load.
+
+rules:
+  - id: unbounded-asyncio-queue
+    message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues).
+    severity: ERROR
+    languages: [python]
+    pattern-either:
+      - pattern: asyncio.Queue()
+      - pattern: asyncio.Queue(maxsize=0)
+    metadata:
+      category: reliability
+      cwe: "CWE-400: Uncontrolled Resource Consumption"
+      tags: [python, reliability]
+      confidence: HIGH
+      source: https://docs.python.org/3/library/asyncio-queue.html

.semgrep/rules/python/unbounded-memory.yml

@@ -0,0 +1,14 @@
+# Unbounded memory growth – data structures without a clear max limit
+# Can lead to OOM under load.
+
+rules:
+  - id: unbounded-asyncio-queue
+    message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues).
+    severity: ERROR
+    languages: [python]
+    pattern-either:
+      - pattern: asyncio.Queue()
+      - pattern: asyncio.Queue(maxsize=0)
+    metadata:
+      category: correctness
+      cwe: "CWE-400: Uncontrolled Resource Consumption"

.semgrep/rules/security/no-claude-directory.yml

@@ -0,0 +1,18 @@
+rules:
+  - id: no-claude-directory-committed
+    message: >
+      .claude/ directory must not be committed to the repository.
+      It contains local Claude Code settings (permissions, worktree paths) that are
+      developer-machine-specific and may expose internal paths or credentials.
+      Add .claude/ to .gitignore instead.
+    severity: ERROR
+    languages: [generic]
+    paths:
+      include:
+        - "/.claude/**"
+        - "/.claude/*"
+    pattern-regex: '[\s\S]+'
+    metadata:
+      category: security
+      tags: [supply-chain, secrets]
+      confidence: HIGH

.trivyignore

@@ -0,0 +1,12 @@
+# LiteLLM Trivy Ignore File
+# CVEs listed here are temporarily allowlisted pending fixes
+
+# Next.js vulnerabilities in UI dashboard (next@14.2.35)
+# Allowlisted: 2026-01-31, 7-day fix timeline
+# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+
+
+# HIGH: DoS via request deserialization
+GHSA-h25m-26qc-wcjf
+
+# MEDIUM: Image Optimizer DoS
+CVE-2025-59471

AGENTS.md

@@ -49,6 +49,29 @@ LiteLLM is a unified interface for 100+ LLMs that:
    - Test provider-specific functionality thoroughly
    - Consider adding load tests for performance-critical changes
 
+### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
+
+1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes**
+   - The only exception is the Tremor Table component and its required Tremor Table sub components.
+
+2. **Use Common Components as much as possible**:
+   - These are usually defined in the `common_components` directory
+   - Use these components as much as possible and avoid building new components unless needed
+
+3. **Testing**:
+   - The codebase uses **Vitest** and **React Testing Library**
+   - **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
+   - **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
+   - **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled
+   - **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present
+   - **Test names must start with "should"**: All test names should follow the pattern `it("should ...")`
+   - **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed
+   - **Structure tests properly**:
+     - First test should verify the component renders successfully
+     - Subsequent tests should focus on functionality and user interactions
+     - Use `waitFor` for async operations that aren't already awaited
+   - **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation
+
 ### IMPORTANT PATTERNS
 
 1. **Function/Tool Calling**:
@@ -86,6 +109,8 @@ Key files:
 - `litellm/proxy/auth/` - Authentication logic
 - `litellm/proxy/management_endpoints/` - Admin API endpoints
 
+**Database (proxy)**: Use Prisma model methods (`prisma_client.db.<model>.upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details.
+
 ## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
 
 LiteLLM supports MCP for agent workflows:
@@ -94,6 +119,29 @@ LiteLLM supports MCP for agent workflows:
 - Support for external MCP servers (Zapier, Jira, Linear, etc.)
 - See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
 
+## RUNNING SCRIPTS
+
+Use `poetry run python script.py` to run Python scripts in the project environment (for non-test files).
+
+## GITHUB TEMPLATES
+
+When opening issues or pull requests, follow these templates:
+
+### Bug Reports (`.github/ISSUE_TEMPLATE/bug_report.yml`)
+- Describe what happened vs. expected behavior
+- Include relevant log output
+- Specify LiteLLM version
+- Indicate if you're part of an ML Ops team (helps with prioritization)
+
+### Feature Requests (`.github/ISSUE_TEMPLATE/feature_request.yml`)
+- Clearly describe the feature
+- Explain motivation and use case with concrete examples
+
+### Pull Requests (`.github/pull_request_template.md`)
+- Add at least 1 test in `tests/litellm/`
+- Ensure `make test-unit` passes
+
+
 ## TESTING CONSIDERATIONS
 
 1. **Provider Tests**: Test against real provider APIs when possible
@@ -128,6 +176,43 @@ LiteLLM supports MCP for agent workflows:
 3. **Rate Limits**: Respect provider rate limits in tests
 4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
 5. **Dependencies**: Keep dependencies minimal and well-justified
+6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
+7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
+8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift)
+
+8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
+
+   **Example of BAD** (hardcoded model checks):
+
+   ```python
+   @staticmethod
+   def _is_effort_supported_model(model: str) -> bool:
+       """Check if the model supports the output_config.effort parameter..."""
+       model_lower = model.lower()
+       if AnthropicConfig._is_claude_4_6_model(model):
+           return True
+       return any(
+           v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
+       )
+   ```
+
+   **Example of GOOD** (config-driven or helper that reads from config):
+
+   ```python
+   if (
+       "claude-3-7-sonnet" in model
+       or AnthropicConfig._is_claude_4_6_model(model)
+       or supports_reasoning(
+           model=model,
+           custom_llm_provider=self.custom_llm_provider,
+       )
+   ):
+       ...
+   ```
+
+   Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
+
+9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history.
 
 ## HELPFUL RESOURCES
 
@@ -141,4 +226,49 @@ LiteLLM supports MCP for agent workflows:
 - Check similar provider implementations
 - Ensure comprehensive test coverage
 - Update documentation appropriately
-- Consider backward compatibility impact 
+- Consider backward compatibility impact
+
+## Cursor Cloud specific instructions
+
+### Environment
+
+- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
+- Python 3.12, Node 22 are pre-installed.
+- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`.
+
+### Running the proxy server
+
+Start the proxy with a config file:
+
+```bash
+poetry run litellm --config dev_config.yaml --port 4000
+```
+
+The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
+
+### Running tests
+
+See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
+
+- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
+- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
+- The `--timeout` pytest flag is NOT available; don't pass it.
+- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
+- Black `--check` may report pre-existing formatting issues; this does not block test runs.
+- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
+
+### Lint
+
+```bash
+cd litellm && poetry run ruff check .
+```
+
+Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
+
+### UI Dashboard development
+
+- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
+- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
+- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
+- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
+- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`

ARCHITECTURE.md

@@ -0,0 +1,398 @@
+# LiteLLM Architecture - LiteLLM SDK + AI Gateway
+
+This document helps contributors understand where to make changes in LiteLLM.
+
+---
+
+## How It Works
+
+The LiteLLM AI Gateway (Proxy) uses the LiteLLM SDK internally for all LLM calls:
+
+```
+OpenAI SDK (client)    ──▶  LiteLLM AI Gateway (proxy/)  ──▶  LiteLLM SDK (litellm/)  ──▶  LLM API
+Anthropic SDK (client) ──▶  LiteLLMAI Gateway (proxy/)  ──▶  LiteLLM SDK (litellm/)  ──▶  LLM API
+Any HTTP client        ──▶  LiteLLMAI Gateway (proxy/)  ──▶  LiteLLM SDK (litellm/)  ──▶  LLM API
+```
+
+The **AI Gateway** adds authentication, rate limiting, budgets, and routing on top of the SDK.
+The **SDK** handles the actual LLM provider calls, request/response transformations, and streaming.
+
+---
+
+## 1. AI Gateway (Proxy) Request Flow
+
+The AI Gateway (`litellm/proxy/`) wraps the SDK with authentication, rate limiting, and management features.
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant ProxyServer as proxy/proxy_server.py
+    participant Auth as proxy/auth/user_api_key_auth.py
+    participant Redis as Redis Cache
+    participant Hooks as proxy/hooks/
+    participant Router as router.py
+    participant Main as main.py + utils.py
+    participant Handler as llms/custom_httpx/llm_http_handler.py
+    participant Transform as llms/{provider}/chat/transformation.py
+    participant Provider as LLM Provider API
+    participant CostCalc as cost_calculator.py
+    participant LoggingObj as litellm_logging.py
+    participant DBWriter as db/db_spend_update_writer.py
+    participant Postgres as PostgreSQL
+
+    %% Request Flow
+    Client->>ProxyServer: POST /v1/chat/completions
+    ProxyServer->>Auth: user_api_key_auth()
+    Auth->>Redis: Check API key cache
+    Redis-->>Auth: Key info + spend limits
+    ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
+    Hooks->>Redis: Check/increment rate limit counters
+    ProxyServer->>Router: route_request()
+    Router->>Main: litellm.acompletion()
+    Main->>Handler: BaseLLMHTTPHandler.completion()
+    Handler->>Transform: ProviderConfig.transform_request()
+    Handler->>Provider: HTTP Request
+    Provider-->>Handler: Response
+    Handler->>Transform: ProviderConfig.transform_response()
+    Transform-->>Handler: ModelResponse
+    Handler-->>Main: ModelResponse
+    
+    %% Cost Attribution (in utils.py wrapper)
+    Main->>LoggingObj: update_response_metadata()
+    LoggingObj->>CostCalc: _response_cost_calculator()
+    CostCalc->>CostCalc: completion_cost(tokens × price)
+    CostCalc-->>LoggingObj: response_cost
+    LoggingObj-->>Main: Set response._hidden_params["response_cost"]
+    Main-->>ProxyServer: ModelResponse (with cost in _hidden_params)
+    
+    %% Response Headers + Async Logging
+    ProxyServer->>ProxyServer: Extract cost from hidden_params
+    ProxyServer->>LoggingObj: async_success_handler()
+    LoggingObj->>Hooks: async_log_success_event()
+    Hooks->>DBWriter: update_database(response_cost)
+    DBWriter->>Redis: Queue spend increment
+    DBWriter->>Postgres: Batch write spend logs (async)
+    ProxyServer-->>Client: ModelResponse + x-litellm-response-cost header
+```
+
+### Proxy Components
+
+```mermaid
+graph TD
+    subgraph "Incoming Request"
+        Client["POST /v1/chat/completions"]
+    end
+
+    subgraph "proxy/proxy_server.py"
+        Endpoint["chat_completion()"]
+    end
+
+    subgraph "proxy/auth/"
+        Auth["user_api_key_auth()"]
+    end
+
+    subgraph "proxy/"
+        PreCall["litellm_pre_call_utils.py"]
+        RouteRequest["route_llm_request.py"]
+    end
+
+    subgraph "litellm/"
+        Router["router.py"]
+        Main["main.py"]
+    end
+
+    subgraph "Infrastructure"
+        DualCache["DualCache<br/>(in-memory + Redis)"]
+        Postgres["PostgreSQL<br/>(keys, teams, spend logs)"]
+    end
+
+    Client --> Endpoint
+    Endpoint --> Auth
+    Auth --> DualCache
+    DualCache -.->|cache miss| Postgres
+    Auth --> PreCall
+    PreCall --> RouteRequest
+    RouteRequest --> Router
+    Router --> DualCache
+    Router --> Main
+    Main --> Client
+```
+
+**Key proxy files:**
+- `proxy/proxy_server.py` - Main API endpoints
+- `proxy/auth/` - Authentication (API keys, JWT, OAuth2)
+- `proxy/hooks/` - Proxy-level callbacks
+- `router.py` - Load balancing, fallbacks
+- `router_strategy/` - Routing algorithms (`lowest_latency.py`, `simple_shuffle.py`, etc.)
+
+**LLM-specific proxy endpoints:**
+
+| Endpoint | Directory | Purpose |
+|----------|-----------|---------|
+| `/v1/messages` | `proxy/anthropic_endpoints/` | Anthropic Messages API |
+| `/vertex-ai/*` | `proxy/vertex_ai_endpoints/` | Vertex AI passthrough |
+| `/gemini/*` | `proxy/google_endpoints/` | Google AI Studio passthrough |
+| `/v1/images/*` | `proxy/image_endpoints/` | Image generation |
+| `/v1/batches` | `proxy/batches_endpoints/` | Batch processing |
+| `/v1/files` | `proxy/openai_files_endpoints/` | File uploads |
+| `/v1/fine_tuning` | `proxy/fine_tuning_endpoints/` | Fine-tuning jobs |
+| `/v1/rerank` | `proxy/rerank_endpoints/` | Reranking |
+| `/v1/responses` | `proxy/response_api_endpoints/` | OpenAI Responses API |
+| `/v1/vector_stores` | `proxy/vector_store_endpoints/` | Vector stores |
+| `/*` (passthrough) | `proxy/pass_through_endpoints/` | Direct provider passthrough |
+
+**Proxy Hooks** (`proxy/hooks/__init__.py`):
+
+| Hook | File | Purpose |
+|------|------|---------|
+| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits |
+| `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
+| `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
+| `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |
+| `litellm_skills` | `proxy/hooks/skills_injection.py` | Skills injection |
+
+To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`.
+
+### Infrastructure Components
+
+The AI Gateway uses external infrastructure for persistence and caching:
+
+```mermaid
+graph LR
+    subgraph "AI Gateway (proxy/)"
+        Proxy["proxy_server.py"]
+        Auth["auth/user_api_key_auth.py"]
+        DBWriter["db/db_spend_update_writer.py<br/>DBSpendUpdateWriter"]
+        InternalCache["utils.py<br/>InternalUsageCache"]
+        CostCallback["hooks/proxy_track_cost_callback.py<br/>_ProxyDBLogger"]
+        Scheduler["APScheduler<br/>ProxyStartupEvent"]
+    end
+
+    subgraph "SDK (litellm/)"
+        Router["router.py<br/>Router.cache (DualCache)"]
+        LLMCache["caching/caching_handler.py<br/>LLMCachingHandler"]
+        CacheClass["caching/caching.py<br/>Cache"]
+    end
+
+    subgraph "Redis (caching/redis_cache.py)"
+        RateLimit["Rate Limit Counters"]
+        SpendQueue["Spend Increment Queue"]
+        KeyCache["API Key Cache"]
+        TPM_RPM["TPM/RPM Tracking"]
+        Cooldowns["Deployment Cooldowns"]
+        LLMResponseCache["LLM Response Cache"]
+    end
+
+    subgraph "PostgreSQL (proxy/schema.prisma)"
+        Keys["LiteLLM_VerificationToken"]
+        Teams["LiteLLM_TeamTable"]
+        SpendLogs["LiteLLM_SpendLogs"]
+        Users["LiteLLM_UserTable"]
+    end
+
+    Auth --> InternalCache
+    InternalCache --> KeyCache
+    InternalCache -.->|cache miss| Keys
+    InternalCache --> RateLimit
+    Router --> TPM_RPM
+    Router --> Cooldowns
+    LLMCache --> CacheClass
+    CacheClass --> LLMResponseCache
+    CostCallback --> DBWriter
+    DBWriter --> SpendQueue
+    DBWriter --> SpendLogs
+    Scheduler --> SpendLogs
+    Scheduler --> Keys
+```
+
+| Component | Purpose | Key Files/Classes |
+|-----------|---------|-------------------|
+| **Redis** | Rate limiting, API key caching, TPM/RPM tracking, cooldowns, LLM response caching, spend queuing | `caching/redis_cache.py` (`RedisCache`), `caching/dual_cache.py` (`DualCache`) |
+| **PostgreSQL** | API keys, teams, users, spend logs | `proxy/utils.py` (`PrismaClient`), `proxy/schema.prisma` |
+| **InternalUsageCache** | Proxy-level cache for rate limits + API keys (in-memory + Redis) | `proxy/utils.py` (`InternalUsageCache`) |
+| **Router.cache** | TPM/RPM tracking, deployment cooldowns, client caching (in-memory + Redis) | `router.py` (`Router.cache: DualCache`) |
+| **LLMCachingHandler** | SDK-level LLM response/embedding caching | `caching/caching_handler.py` (`LLMCachingHandler`), `caching/caching.py` (`Cache`) |
+| **DBSpendUpdateWriter** | Batches spend updates to reduce DB writes | `proxy/db/db_spend_update_writer.py` (`DBSpendUpdateWriter`) |
+| **Cost Tracking** | Calculates and logs response costs | `proxy/hooks/proxy_track_cost_callback.py` (`_ProxyDBLogger`) |
+
+**Background Jobs** (APScheduler, initialized in `proxy/proxy_server.py` → `ProxyStartupEvent.initialize_scheduled_background_jobs()`):
+
+| Job | Interval | Purpose | Key Files |
+|-----|----------|---------|-----------|
+| `update_spend` | 60s | Batch write spend logs to PostgreSQL | `proxy/db/db_spend_update_writer.py` |
+| `reset_budget` | 10-12min | Reset budgets for keys/users/teams | `proxy/management_helpers/budget_reset_job.py` |
+| `add_deployment` | 10s | Sync new model deployments from DB | `proxy/proxy_server.py` (`ProxyConfig`) |
+| `cleanup_old_spend_logs` | cron/interval | Delete old spend logs | `proxy/management_helpers/spend_log_cleanup.py` |
+| `check_batch_cost` | 30min | Calculate costs for batch jobs | `proxy/management_helpers/check_batch_cost_job.py` |
+| `check_responses_cost` | 30min | Calculate costs for responses API | `proxy/management_helpers/check_responses_cost_job.py` |
+| `process_rotations` | 1hr | Auto-rotate API keys | `proxy/management_helpers/key_rotation_manager.py` |
+| `_run_background_health_check` | continuous | Health check model deployments | `proxy/proxy_server.py` |
+| `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
+| `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
+
+**Cost Attribution Flow:**
+1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes
+2. `update_response_metadata()` (`llm_response_utils/response_metadata.py`) is called
+3. `logging_obj._response_cost_calculator()` (`litellm_logging.py`) calculates cost via `litellm.completion_cost()` (`cost_calculator.py`)
+4. Cost is stored in `response._hidden_params["response_cost"]`
+5. `proxy/common_request_processing.py` extracts cost from `hidden_params` and adds to response headers (`x-litellm-response-cost`)
+6. `logging_obj.async_success_handler()` triggers callbacks including `_ProxyDBLogger.async_log_success_event()`
+7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis
+8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s
+
+---
+
+## 2. SDK Request Flow
+
+The SDK (`litellm/`) provides the core LLM calling functionality used by both direct SDK users and the AI Gateway.
+
+```mermaid
+graph TD
+    subgraph "SDK Entry Points"
+        Completion["litellm.completion()"]
+        Messages["litellm.messages()"]
+    end
+
+    subgraph "main.py"
+        Main["completion()<br/>acompletion()"]
+    end
+
+    subgraph "utils.py"
+        GetProvider["get_llm_provider()"]
+    end
+
+    subgraph "llms/custom_httpx/"
+        Handler["llm_http_handler.py<br/>BaseLLMHTTPHandler"]
+        HTTP["http_handler.py<br/>HTTPHandler / AsyncHTTPHandler"]
+    end
+
+    subgraph "llms/{provider}/chat/"
+        TransformReq["transform_request()"]
+        TransformResp["transform_response()"]
+    end
+
+    subgraph "litellm_core_utils/"
+        Streaming["streaming_handler.py"]
+    end
+
+    subgraph "integrations/ (async, off main thread)"
+        Callbacks["custom_logger.py<br/>Langfuse, Datadog, etc."]
+    end
+
+    Completion --> Main
+    Messages --> Main
+    Main --> GetProvider
+    GetProvider --> Handler
+    Handler --> TransformReq
+    TransformReq --> HTTP
+    HTTP --> Provider["LLM Provider API"]
+    Provider --> HTTP
+    HTTP --> TransformResp
+    TransformResp --> Streaming
+    Streaming --> Response["ModelResponse"]
+    Response -.->|async| Callbacks
+```
+
+**Key SDK files:**
+- `main.py` - Entry points: `completion()`, `acompletion()`, `embedding()`
+- `utils.py` - `get_llm_provider()` resolves model → provider
+- `llms/custom_httpx/llm_http_handler.py` - Central HTTP orchestrator
+- `llms/custom_httpx/http_handler.py` - Low-level HTTP client
+- `llms/{provider}/chat/transformation.py` - Provider-specific transformations
+- `litellm_core_utils/streaming_handler.py` - Streaming response handling
+- `integrations/` - Async callbacks (Langfuse, Datadog, etc.)
+
+---
+
+## 3. Translation Layer
+
+When a request comes in, it goes through a **translation layer** that converts between API formats.
+Each translation is isolated in its own file, making it easy to test and modify independently.
+
+### Where to find translations
+
+| Incoming API | Provider | Translation File |
+|--------------|----------|------------------|
+| `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` |
+| `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` |
+| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` |
+| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
+| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
+| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
+| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
+| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
+| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
+| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
+
+### Example: Debugging prompt caching
+
+If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works:
+
+1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py`
+2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py`
+3. Compare how each handles `cache_control` in `transform_request()`
+
+### How translations work
+
+Each provider has a `Config` class that inherits from `BaseConfig` (`llms/base_llm/chat/transformation.py`):
+
+```python
+class ProviderConfig(BaseConfig):
+    def transform_request(self, model, messages, optional_params, litellm_params, headers):
+        # Convert OpenAI format → Provider format
+        return {"messages": transformed_messages, ...}
+    
+    def transform_response(self, model, raw_response, model_response, logging_obj, ...):
+        # Convert Provider format → OpenAI format
+        return ModelResponse(choices=[...], usage=Usage(...))
+```
+
+The `BaseLLMHTTPHandler` (`llms/custom_httpx/llm_http_handler.py`) calls these methods - you never need to modify the handler itself.
+
+---
+
+## 4. Adding/Modifying Providers
+
+### To add a new provider:
+
+1. Create `llms/{provider}/chat/transformation.py`
+2. Implement `Config` class with `transform_request()` and `transform_response()`
+3. Add tests in `tests/llm_translation/test_{provider}.py`
+
+### To add a feature (e.g., prompt caching):
+
+1. Find the translation file from the table above
+2. Modify `transform_request()` to handle the new parameter
+3. Add unit tests that verify the transformation
+
+### Testing checklist
+
+When adding a feature, verify it works across all paths:
+
+| Test | File Pattern |
+|------|--------------|
+| OpenAI passthrough | `tests/llm_translation/test_openai*.py` |
+| Anthropic direct | `tests/llm_translation/test_anthropic*.py` |
+| Bedrock Invoke | `tests/llm_translation/test_bedrock*.py` |
+| Bedrock Converse | `tests/llm_translation/test_bedrock*converse*.py` |
+| Vertex AI | `tests/llm_translation/test_vertex*.py` |
+| Gemini | `tests/llm_translation/test_gemini*.py` |
+
+### Unit testing translations
+
+Translations are designed to be unit testable without making API calls:
+
+```python
+from litellm.llms.bedrock.chat.converse_transformation import BedrockConverseConfig
+
+def test_prompt_caching_transform():
+    config = BedrockConverseConfig()
+    result = config.transform_request(
+        model="anthropic.claude-3-opus",
+        messages=[{"role": "user", "content": "test", "cache_control": {"type": "ephemeral"}}],
+        optional_params={},
+        litellm_params={},
+        headers={}
+    )
+    assert "cachePoint" in str(result)  # Verify cache_control was translated
+```

CLAUDE.md

@@ -25,6 +25,25 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 - `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
 - `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
 
+### Running Scripts
+- `poetry run python script.py` - Run Python scripts (use for non-test files)
+
+### GitHub Issue & PR Templates
+When contributing to the project, use the appropriate templates:
+
+**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
+- Describe what happened vs. what you expected
+- Include relevant log output
+- Specify your LiteLLM version
+
+**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
+- Describe the feature clearly
+- Explain the motivation and use case
+
+**Pull Requests** (`.github/pull_request_template.md`):
+- Add at least 1 test in `tests/litellm/`
+- Ensure `make test-unit` passes
+
 ## Architecture Overview
 
 LiteLLM is a unified interface for 100+ LLM providers with two main components:
@@ -71,19 +90,75 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Pydantic v2 for data validation
 - Async/await patterns throughout
 - Type hints required for all public APIs
+- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
+- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear.
+- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with.
+- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller.
+- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing.
 
 ### Testing Strategy
 - Unit tests in `tests/test_litellm/`
 - Integration tests for each provider in `tests/llm_translation/`
 - Proxy tests in `tests/proxy_unit_tests/`
 - Load tests in `tests/load_tests/`
+- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
+- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs.
+- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide.
+
+### UI / Backend Consistency
+- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
+
+### MCP OAuth / OpenAPI Transport Mapping
+- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
+- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
+- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
+- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it.
+
+### MCP Credential Storage
+- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string).
+- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair.
+- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp.
+- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints.
+
+### Browser Storage Safety (UI)
+- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS).
+- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files.
 
 ### Database Migrations
 - Prisma handles schema migrations
 - Migration files auto-generated with `prisma migrate dev`
 - Always test migrations against both PostgreSQL and SQLite
 
+### Proxy database access
+- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
+- Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
+- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory.
+- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks.
+- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing.
+- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
+- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
+- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
+- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
+
+### Setup Wizard (`litellm/setup_wizard.py`)
+- The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).
+- Use `litellm.utils.check_valid_key(model, api_key)` for credential validation — never roll a custom completion call.
+- Do not hardcode provider env-key names or model lists that already exist in the codebase. Add a `test_model` field to each provider entry to drive `check_valid_key`; set it to `None` for providers that can't be validated with a single API key (Azure, Bedrock, Ollama).
+
 ### Enterprise Features
 - Enterprise-specific code in `enterprise/` directory
 - Optional features enabled via environment variables
-- Separate licensing and authentication for enterprise features
+- Separate licensing and authentication for enterprise features
+
+### HTTP Client Cache Safety
+- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
+
+### Troubleshooting: DB schema out of sync after proxy restart
+`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
+
+**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
+
+**Fix options:**
+1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
+2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
+3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

CONTRIBUTING.md

@@ -7,11 +7,20 @@ Thank you for your interest in contributing to LiteLLM! We welcome contributions
 Here are the core requirements for any PR submitted to LiteLLM:
 
 - [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
+- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+
+#### Proxy (Backend) PRs
+
 - [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
 - [ ] **Ensure your PR passes all checks**:
   - [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
   - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
-- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+
+#### UI PRs
+
+- [ ] **Ensure the UI builds successfully** - `npm run build`
+- [ ] **Ensure all UI unit tests pass** - `npm run test`
+- [ ] **Add tests for new components or logic** - If you are adding a new component or new logic, add corresponding tests
 
 ## **Contributor License Agreement (CLA)**
 
@@ -24,8 +33,9 @@ Before contributing code to LiteLLM, you must sign our [Contributor License Agre
 ### 1. Setup Your Local Development Environment
 
 ```bash
-# Clone the repository
-git clone https://github.com/BerriAI/litellm.git
+# Fork the repository on GitHub (click the Fork button at https://github.com/BerriAI/litellm)
+# Then clone your fork locally
+git clone https://github.com/YOUR_USERNAME/litellm.git
 cd litellm
 
 # Create a new branch for your feature
@@ -244,6 +254,43 @@ docker run \
     --config /app/config.yaml --detailed_debug
 ```
 
+## UI Development
+
+### 1. Setup Your Local UI Development Environment
+
+```bash
+# Clone the repo (if you haven't already)
+git clone https://github.com/YOUR_USERNAME/litellm.git
+cd litellm
+
+# Navigate to the UI dashboard directory
+cd ui/litellm-dashboard
+
+# Install dependencies
+npm install
+
+# Start the development server
+npm run dev
+```
+
+### 2. Adding UI Tests
+
+If you are adding a **new component** or **new logic**, you must add corresponding tests.
+
+### 3. Running UI Unit Tests
+
+```bash
+npm run test
+```
+
+### 4. Building the UI
+
+Ensure the UI builds successfully before submitting your PR:
+
+```bash
+npm run build
+```
+
 ## Submitting Your PR
 
 1. **Push your branch**: `git push origin your-feature-branch`
@@ -258,7 +305,7 @@ docker run \
 If you need help:
 
 - 💬 [Join our Discord](https://discord.gg/wuPM9dRgDw)
-- 💬 [Join our Slack](https://join.slack.com/share/enQtOTE0ODczMzk2Nzk4NC01YjUxNjY2YjBlYTFmNDRiZTM3NDFiYTM3MzVkODFiMDVjOGRjMmNmZTZkZTMzOWQzZGQyZWIwYjQ0MWExYmE3)
+- 💬 [Join our Slack](https://www.litellm.ai/support)
 - 📧 Email us: ishaan@berri.ai / krrish@berri.ai
 - 🐛 [Create an issue](https://github.com/BerriAI/litellm/issues/new)
 

Dockerfile

@@ -1,8 +1,9 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
+
 # Builder stage
 FROM $LITELLM_BUILD_IMAGE AS builder
 
@@ -12,17 +13,16 @@ WORKDIR /app
 USER root
 
 # Install build dependencies
-RUN apk add --no-cache gcc python3-dev openssl openssl-dev
+RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev
 
-
-RUN pip install --upgrade pip && \
-    pip install build
+RUN python -m pip install build
 
 # Copy the current directory contents into the container at /app
 COPY . .
 
 # Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+# Convert Windows line endings to Unix and make executable
+RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Build the package
 RUN rm -rf dist/* && python -m build
@@ -39,7 +39,7 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.9.0 --no-cache-dir
+RUN pip install PyJWT==2.12.0 --no-cache-dir
 
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
@@ -47,8 +47,38 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 # Ensure runtime stage runs as root
 USER root
 
-# Install runtime dependencies
-RUN apk add --no-cache openssl tzdata
+# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
+    # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
+    # levels inside its dependency tree. `npm install -g <pkg>` only creates a
+    # SEPARATE global package, it does NOT replace npm's internal copies.
+    # We must find and replace EVERY copy inside npm's directory.
+    GLOBAL="$(npm root -g)" && \
+    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    # SECURITY FIX: patch npm's own package.json metadata so scanners see the
+    # actual installed versions instead of the stale declared dependencies.
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    # Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
+    # no longer visible to image scanners.  The globally installed npm@latest
+    # at /usr/local/lib/node_modules/npm/ remains fully functional.
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -62,13 +92,44 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
-# Install semantic_router and aurelio-sdk using script
-RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
+RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
+    if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
 
-# Generate prisma client
-RUN prisma generate
-RUN chmod +x docker/entrypoint.sh
-RUN chmod +x docker/prod_entrypoint.sh
+# Remove test files and keys from dependencies
+RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
+    find /usr/lib -type d -path "*/tornado/test" -delete
+
+# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
+# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
+# Patch every copy of tar, glob, and brace-expansion inside that tree.
+RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done
+
+# Install semantic_router and aurelio-sdk using script
+# Convert Windows line endings to Unix and make executable
+RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+
+# Generate prisma client using the correct schema
+RUN prisma generate --schema=./litellm/proxy/schema.prisma
+# Convert Windows line endings to Unix for entrypoint scripts
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
+RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
 EXPOSE 4000/tcp
 

GEMINI.md

@@ -25,6 +25,25 @@ This file provides guidance to Gemini when working with code in this repository.
 - `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
 - `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
 
+### Running Scripts
+- `poetry run python script.py` - Run Python scripts (use for non-test files)
+
+### GitHub Issue & PR Templates
+When contributing to the project, use the appropriate templates:
+
+**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
+- Describe what happened vs. what you expected
+- Include relevant log output
+- Specify your LiteLLM version
+
+**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
+- Describe the feature clearly
+- Explain the motivation and use case
+
+**Pull Requests** (`.github/pull_request_template.md`):
+- Add at least 1 test in `tests/litellm/`
+- Ensure `make test-unit` passes
+
 ## Architecture Overview
 
 LiteLLM is a unified interface for 100+ LLM providers with two main components:

Makefile

@@ -1,7 +1,12 @@
 # LiteLLM Makefile
 # Simple Makefile for running tests and basic development tasks
 
-.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
+.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
+	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
+	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
+	info lint lint-dev format \
+	install-dev install-proxy-dev install-test-deps \
+	install-helm-unittest check-circular-imports check-import-safety
 
 # Default target
 help:
@@ -22,9 +27,26 @@ help:
 	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
 	@echo "  make test-unit          - Run unit tests (tests/test_litellm)"
+	@echo "  make test-unit-llms     - Run LLM provider tests (~225 files)"
+	@echo "  make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
+	@echo "  make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
+	@echo "  make test-unit-proxy-misc - Run proxy misc tests (~77 files)"
+	@echo "  make test-unit-integrations - Run integration tests (~60 files)"
+	@echo "  make test-unit-core-utils - Run core utils tests (~32 files)"
+	@echo "  make test-unit-other    - Run other tests (caching, responses, etc., ~69 files)"
+	@echo "  make test-unit-root     - Run root-level tests (~34 files)"
+	@echo "  make test-proxy-unit-a  - Run proxy_unit_tests (a-o, ~20 files)"
+	@echo "  make test-proxy-unit-b  - Run proxy_unit_tests (p-z, ~28 files)"
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
 
+# Keep PIP simple for edge cases:
+PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip")
+
+# Show info
+info:
+	@echo "PIP: $(PIP)"
+
 # Installation targets
 install-dev:
 	poetry install --with dev
@@ -34,18 +56,19 @@ install-proxy-dev:
 
 # CI-compatible installations (matches GitHub workflows exactly)
 install-dev-ci:
-	pip install openai==1.99.5
+	$(PIP) install openai==2.8.0
 	poetry install --with dev
-	pip install openai==1.99.5
+	$(PIP) install openai==2.8.0
 
 install-proxy-dev-ci:
 	poetry install --with dev,proxy-dev --extras proxy
-	pip install openai==1.99.5
+	$(PIP) install openai==2.8.0
 
 install-test-deps: install-proxy-dev
-	poetry run pip install "pytest-retry==1.6.3"
-	poetry run pip install pytest-xdist
-	cd enterprise && python -m pip install -e . && cd ..
+	poetry run $(PIP) install "pytest-retry==1.6.3"
+	poetry run $(PIP) install pytest-xdist
+	poetry run $(PIP) install openapi-core
+	cd enterprise && poetry run $(PIP) install -e . && cd ..
 
 install-helm-unittest:
 	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
@@ -61,8 +84,40 @@ format-check: install-dev
 lint-ruff: install-dev
 	cd litellm && poetry run ruff check . && cd ..
 
+# faster linter for developing ...
+# inspiration from:
+# https://github.com/astral-sh/ruff/discussions/10977
+# https://github.com/astral-sh/ruff/discussions/4049
+lint-format-changed: install-dev
+	@git diff origin/main --unified=0 --no-color -- '*.py' | \
+	perl -ne '\
+		if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
+		if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
+			$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
+			print "$$file:$$start:1-$$end:999\n"; \
+		}' | \
+	while read range; do \
+		file="$${range%%:*}"; \
+		lines="$${range#*:}"; \
+		echo "Formatting $$file (lines $$lines)"; \
+		poetry run ruff format --range "$$lines" "$$file"; \
+	done
+
+lint-ruff-dev: install-dev
+	@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
+	cd litellm && \
+	(poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \
+	poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
+	cd .. ; \
+	rm -f "$$tmpfile"
+
+lint-ruff-FULL-dev: install-dev
+	@files=$$(git diff --name-only origin/main -- '*.py'); \
+	if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \
+	else echo "No changed .py files to check."; fi
+
 lint-mypy: install-dev
-	poetry run pip install types-requests types-setuptools types-redis types-PyYAML
+	poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML
 	cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
 
 lint-black: format-check
@@ -71,11 +126,14 @@ check-circular-imports: install-dev
 	cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
 
 check-import-safety: install-dev
-	poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+	@poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
 
 # Combined linting (matches test-linting.yml workflow)
 lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
 
+# Faster linting for local development (only checks changed code)
+lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
+
 # Testing targets
 test:
 	poetry run pytest tests/
@@ -83,6 +141,38 @@ test:
 test-unit: install-test-deps
 	poetry run pytest tests/test_litellm -x -vv -n 4
 
+# Matrix test targets (matching CI workflow groups)
+test-unit-llms: install-test-deps
+	poetry run pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-guardrails: install-test-deps
+	poetry run pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-core: install-test-deps
+	poetry run pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-misc: install-test-deps
+	poetry run pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
+
+test-unit-integrations: install-test-deps
+	poetry run pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
+
+test-unit-core-utils: install-test-deps
+	poetry run pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
+
+test-unit-other: install-test-deps
+	poetry run pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
+
+test-unit-root: install-test-deps
+	poetry run pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
+
+# Proxy unit tests (tests/proxy_unit_tests split alphabetically)
+test-proxy-unit-a: install-test-deps
+	poetry run pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
+
+test-proxy-unit-b: install-test-deps
+	poetry run pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
+
 test-integration:
 	poetry run pytest tests/ -k "not test_litellm"
 
@@ -100,4 +190,4 @@ test-llm-translation-single: install-test-deps
 	@mkdir -p test-results
 	poetry run pytest tests/llm_translation/$(FILE) \
 		--junitxml=test-results/junit.xml \
-		-v --tb=short --maxfail=100 --timeout=300
+		-v --tb=short --maxfail=100 --timeout=300

README.md

@@ -2,16 +2,16 @@
         🚅 LiteLLM
     </h1>
     <p align="center">
+        <p align="center">Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.]
+        </p>
         <p align="center">
         <a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
         <a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
           <img src="https://railway.app/button.svg" alt="Deploy on Railway">
         </a>
         </p>
-        <p align="center">Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]
-        <br>
     </p>
-<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (LLM Gateway)</a> | <a href="https://docs.litellm.ai/docs/hosted" target="_blank"> Hosted Proxy (Preview)</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
+<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
 <h4 align="center">
     <a href="https://pypi.org/project/litellm/" target="_blank">
         <img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
@@ -28,31 +28,22 @@
     <a href="https://www.litellm.ai/support">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
     </a>
+    <a href="https://codspeed.io/BerriAI/litellm?utm_source=badge">
+        <img src="https://img.shields.io/endpoint?url=https://codspeed.io/badge.json" alt="CodSpeed"/>
+    </a>
 </h4>
 
-LiteLLM manages:
+<img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
 
-- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
-- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
-- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
-- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
 
-[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
-[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
+## Use LiteLLM for
 
-🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
+<details open>
+<summary><b>LLMs</b> - Call 100+ LLMs (Python SDK + AI Gateway)</summary>
 
-Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
+[**All Supported Endpoints**](https://docs.litellm.ai/docs/supported_endpoints) - `/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, `/rerank`, `/a2a`, `/messages` and more.
 
-# Usage ([**Docs**](https://docs.litellm.ai/docs/))
-
-> [!IMPORTANT]
-> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
-> LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
-
-<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
-  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
-</a>
+### Python SDK
 
 ```shell
 pip install litellm
@@ -62,306 +53,357 @@ pip install litellm
 from litellm import completion
 import os
 
-## set ENV variables
 os.environ["OPENAI_API_KEY"] = "your-openai-key"
 os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
 
-messages = [{ "content": "Hello, how are you?","role": "user"}]
+# OpenAI
+response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hello!"}])
 
-# openai call
-response = completion(model="openai/gpt-4o", messages=messages)
-
-# anthropic call
-response = completion(model="anthropic/claude-sonnet-4-20250514", messages=messages)
-print(response)
+# Anthropic  
+response = completion(model="anthropic/claude-sonnet-4-20250514", messages=[{"role": "user", "content": "Hello!"}])
 ```
 
-### Response (OpenAI Format)
+### AI Gateway (Proxy Server)
 
-```json
-{
-    "id": "chatcmpl-1214900a-6cdd-4148-b663-b5e2f642b4de",
-    "created": 1751494488,
-    "model": "claude-sonnet-4-20250514",
-    "object": "chat.completion",
-    "system_fingerprint": null,
-    "choices": [
-        {
-            "finish_reason": "stop",
-            "index": 0,
-            "message": {
-                "content": "Hello! I'm doing well, thank you for asking. I'm here and ready to help with whatever you'd like to discuss or work on. How are you doing today?",
-                "role": "assistant",
-                "tool_calls": null,
-                "function_call": null
-            }
-        }
-    ],
-    "usage": {
-        "completion_tokens": 39,
-        "prompt_tokens": 13,
-        "total_tokens": 52,
-        "completion_tokens_details": null,
-        "prompt_tokens_details": {
-            "audio_tokens": null,
-            "cached_tokens": 0
-        },
-        "cache_creation_input_tokens": 0,
-        "cache_read_input_tokens": 0
-    }
-}
-```
-
-Call any model supported by a provider, with `model=<provider_name>/<model_name>`. There might be provider-specific details here, so refer to [provider docs for more information](https://docs.litellm.ai/docs/providers)
-
-## Async ([Docs](https://docs.litellm.ai/docs/completion/stream#async-completion))
-
-```python
-from litellm import acompletion
-import asyncio
-
-async def test_get_response():
-    user_message = "Hello, how are you?"
-    messages = [{"content": user_message, "role": "user"}]
-    response = await acompletion(model="openai/gpt-4o", messages=messages)
-    return response
-
-response = asyncio.run(test_get_response())
-print(response)
-```
-
-## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
-
-liteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
-Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
-
-```python
-from litellm import completion
-response = completion(model="openai/gpt-4o", messages=messages, stream=True)
-for part in response:
-    print(part.choices[0].delta.content or "")
-
-# claude sonnet 4
-response = completion('anthropic/claude-sonnet-4-20250514', messages, stream=True)
-for part in response:
-    print(part)
-```
-
-### Response chunk (OpenAI Format)
-
-```json
-{
-    "id": "chatcmpl-fe575c37-5004-4926-ae5e-bfbc31f356ca",
-    "created": 1751494808,
-    "model": "claude-sonnet-4-20250514",
-    "object": "chat.completion.chunk",
-    "system_fingerprint": null,
-    "choices": [
-        {
-            "finish_reason": null,
-            "index": 0,
-            "delta": {
-                "provider_specific_fields": null,
-                "content": "Hello",
-                "role": "assistant",
-                "function_call": null,
-                "tool_calls": null,
-                "audio": null
-            },
-            "logprobs": null
-        }
-    ],
-    "provider_specific_fields": null,
-    "stream_options": null,
-    "citations": null
-}
-```
-
-## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
-
-LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
-
-```python
-from litellm import completion
-
-## set env variables for logging tools (when using MLflow, no API key set up is required)
-os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
-os.environ["HELICONE_API_KEY"] = "your-helicone-auth-key"
-os.environ["LANGFUSE_PUBLIC_KEY"] = ""
-os.environ["LANGFUSE_SECRET_KEY"] = ""
-os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
-
-os.environ["OPENAI_API_KEY"] = "your-openai-key"
-
-# set callbacks
-litellm.success_callback = ["lunary", "mlflow", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
-
-#openai call
-response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
-```
-
-# LiteLLM Proxy Server (LLM Gateway) - ([Docs](https://docs.litellm.ai/docs/simple_proxy))
-
-Track spend + Load Balance across multiple projects
-
-[Hosted Proxy (Preview)](https://docs.litellm.ai/docs/hosted)
-
-The proxy provides:
-
-1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
-2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class)
-3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend)
-4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits)
-
-## 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/)
-
-
-## Quick Start Proxy - CLI
+[**Getting Started - E2E Tutorial**](https://docs.litellm.ai/docs/proxy/docker_quick_start) - Setup virtual keys, make your first request
 
 ```shell
 pip install 'litellm[proxy]'
+litellm --model gpt-4o
 ```
 
-### Step 1: Start litellm proxy
-
-```shell
-$ litellm --model huggingface/bigcode/starcoder
-
-#INFO: Proxy running on http://0.0.0.0:4000
-```
-
-### Step 2: Make ChatCompletions Request to Proxy
-
-
-> [!IMPORTANT]
-> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
-
 ```python
-import openai # openai v1.0.0+
-client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url
-# request sent to model set on litellm proxy, `litellm --model`
-response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
-    {
-        "role": "user",
-        "content": "this is a test request, write a short poem"
-    }
-])
+import openai
 
-print(response)
+client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000")
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Hello!"}]
+)
 ```
 
-## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys))
+[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
 
-Connect the proxy with a Postgres DB to create proxy keys
+</details>
+
+<details>
+<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
+
+[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
+
+### Python SDK - A2A Protocol
+
+```python
+from litellm.a2a_protocol import A2AClient
+from a2a.types import SendMessageRequest, MessageSendParams
+from uuid import uuid4
+
+client = A2AClient(base_url="http://localhost:10001")
+
+request = SendMessageRequest(
+    id=str(uuid4()),
+    params=MessageSendParams(
+        message={
+            "role": "user",
+            "parts": [{"kind": "text", "text": "Hello!"}],
+            "messageId": uuid4().hex,
+        }
+    )
+)
+response = await client.send_message(request)
+```
+
+### AI Gateway (Proxy Server)
+
+**Step 1.** [Add your Agent to the AI Gateway](https://docs.litellm.ai/docs/a2a#adding-your-agent)
+
+**Step 2.** Call Agent via A2A SDK
+
+```python
+from a2a.client import A2ACardResolver, A2AClient
+from a2a.types import MessageSendParams, SendMessageRequest
+from uuid import uuid4
+import httpx
+
+base_url = "http://localhost:4000/a2a/my-agent"  # LiteLLM proxy + agent name
+headers = {"Authorization": "Bearer sk-1234"}    # LiteLLM Virtual Key
+
+async with httpx.AsyncClient(headers=headers) as httpx_client:
+    resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
+    agent_card = await resolver.get_agent_card()
+    client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
+
+    request = SendMessageRequest(
+        id=str(uuid4()),
+        params=MessageSendParams(
+            message={
+                "role": "user",
+                "parts": [{"kind": "text", "text": "Hello!"}],
+                "messageId": uuid4().hex,
+            }
+        )
+    )
+    response = await client.send_message(request)
+```
+
+[**Docs: A2A Agent Gateway**](https://docs.litellm.ai/docs/a2a)
+
+</details>
+
+<details>
+<summary><b>MCP Tools</b> - Connect MCP servers to any LLM (Python SDK + AI Gateway)</summary>
+
+### Python SDK - MCP Bridge
+
+```python
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+from litellm import experimental_mcp_client
+import litellm
+
+server_params = StdioServerParameters(command="python", args=["mcp_server.py"])
+
+async with stdio_client(server_params) as (read, write):
+    async with ClientSession(read, write) as session:
+        await session.initialize()
+
+        # Load MCP tools in OpenAI format
+        tools = await experimental_mcp_client.load_mcp_tools(session=session, format="openai")
+
+        # Use with any LiteLLM model
+        response = await litellm.acompletion(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "What's 3 + 5?"}],
+            tools=tools
+        )
+```
+
+### AI Gateway - MCP Gateway
+
+**Step 1.** [Add your MCP Server to the AI Gateway](https://docs.litellm.ai/docs/mcp#adding-your-mcp)
+
+**Step 2.** Call MCP tools via `/chat/completions`
 
 ```bash
-# Get the code
-git clone https://github.com/BerriAI/litellm
-
-# Go to folder
-cd litellm
-
-# Add the master key - you can change this after setup
-echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
-
-# Add the litellm salt key - you cannot change this after adding a model
-# It is used to encrypt / decrypt your LLM API Key credentials
-# We recommend - https://1password.com/password-generator/
-# password generator to get a random hash for litellm salt key
-echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
-
-source .env
-
-# Start
-docker-compose up
+curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [{"role": "user", "content": "Summarize the latest open PR"}],
+    "tools": [{
+      "type": "mcp",
+      "server_url": "litellm_proxy/mcp/github",
+      "server_label": "github_mcp",
+      "require_approval": "never"
+    }]
+  }'
 ```
 
+### Use with Cursor IDE
 
-UI on `/ui` on your proxy server
-![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033)
-
-Set budgets and rate limits across multiple projects
-`POST /key/generate`
-
-### Request
-
-```shell
-curl 'http://0.0.0.0:4000/key/generate' \
---header 'Authorization: Bearer sk-1234' \
---header 'Content-Type: application/json' \
---data-raw '{"models": ["gpt-3.5-turbo", "gpt-4", "claude-2"], "duration": "20m","metadata": {"user": "ishaan@berri.ai", "team": "core-infra"}}'
-```
-
-### Expected Response
-
-```shell
+```json
 {
-    "key": "sk-kdEXbIqZRwEeEiHwdg7sFA", # Bearer token
-    "expires": "2023-11-19T01:38:25.838000+00:00" # datetime object
+  "mcpServers": {
+    "LiteLLM": {
+      "url": "http://localhost:4000/mcp/",
+      "headers": {
+        "x-litellm-api-key": "Bearer sk-1234"
+      }
+    }
+  }
 }
 ```
 
-## Supported Providers ([Docs](https://docs.litellm.ai/docs/providers))
+[**Docs: MCP Gateway**](https://docs.litellm.ai/docs/mcp)
 
-| Provider                                                                            | [Completion](https://docs.litellm.ai/docs/#basic-usage) | [Streaming](https://docs.litellm.ai/docs/completion/stream#streaming-responses) | [Async Completion](https://docs.litellm.ai/docs/completion/stream#async-completion) | [Async Streaming](https://docs.litellm.ai/docs/completion/stream#async-streaming) | [Async Embedding](https://docs.litellm.ai/docs/embedding/supported_embedding) | [Async Image Generation](https://docs.litellm.ai/docs/image_generation) |
-|-------------------------------------------------------------------------------------|---------------------------------------------------------|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------|-------------------------------------------------------------------------------|-------------------------------------------------------------------------|
-| [openai](https://docs.litellm.ai/docs/providers/openai)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             | ✅                                                                       |
-| [Meta - Llama API](https://docs.litellm.ai/docs/providers/meta_llama)                               | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                              |                                                                        |
-| [azure](https://docs.litellm.ai/docs/providers/azure)                               | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             | ✅                                                                       |
-| [AI/ML API](https://docs.litellm.ai/docs/providers/aiml)                               | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             | ✅                                                                       |
-| [aws - sagemaker](https://docs.litellm.ai/docs/providers/aws_sagemaker)             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
-| [aws - bedrock](https://docs.litellm.ai/docs/providers/bedrock)                     | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
-| [google - vertex_ai](https://docs.litellm.ai/docs/providers/vertex)                 | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             | ✅                                                                       |
-| [google - palm](https://docs.litellm.ai/docs/providers/palm)                        | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [google AI Studio - gemini](https://docs.litellm.ai/docs/providers/gemini)          | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [mistral ai api](https://docs.litellm.ai/docs/providers/mistral)                    | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
-| [cloudflare AI Workers](https://docs.litellm.ai/docs/providers/cloudflare_workers)  | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [CompactifAI](https://docs.litellm.ai/docs/providers/compactifai)                   | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [cohere](https://docs.litellm.ai/docs/providers/cohere)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
-| [anthropic](https://docs.litellm.ai/docs/providers/anthropic)                       | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [empower](https://docs.litellm.ai/docs/providers/empower)                    | ✅                                                      | ✅                                                                              | ✅                                                                                  | ✅                                                                                |
-| [huggingface](https://docs.litellm.ai/docs/providers/huggingface)                   | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
-| [replicate](https://docs.litellm.ai/docs/providers/replicate)                       | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [together_ai](https://docs.litellm.ai/docs/providers/togetherai)                    | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [openrouter](https://docs.litellm.ai/docs/providers/openrouter)                     | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [ai21](https://docs.litellm.ai/docs/providers/ai21)                                 | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [baseten](https://docs.litellm.ai/docs/providers/baseten)                           | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [vllm](https://docs.litellm.ai/docs/providers/vllm)                                 | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [nlp_cloud](https://docs.litellm.ai/docs/providers/nlp_cloud)                       | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [aleph alpha](https://docs.litellm.ai/docs/providers/aleph_alpha)                   | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [petals](https://docs.litellm.ai/docs/providers/petals)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [ollama](https://docs.litellm.ai/docs/providers/ollama)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
-| [deepinfra](https://docs.litellm.ai/docs/providers/deepinfra)                       | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [perplexity-ai](https://docs.litellm.ai/docs/providers/perplexity)                  | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [Groq AI](https://docs.litellm.ai/docs/providers/groq)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [Deepseek](https://docs.litellm.ai/docs/providers/deepseek)                         | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [anyscale](https://docs.litellm.ai/docs/providers/anyscale)                         | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [IBM - watsonx.ai](https://docs.litellm.ai/docs/providers/watsonx)                  | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
-| [voyage ai](https://docs.litellm.ai/docs/providers/voyage)                          |                                                         |                                                                                 |                                                                                     |                                                                                   | ✅                                                                             |                                                                         |
-| [xinference [Xorbits Inference]](https://docs.litellm.ai/docs/providers/xinference) |                                                         |                                                                                 |                                                                                     |                                                                                   | ✅                                                                             |                                                                         |
-| [FriendliAI](https://docs.litellm.ai/docs/providers/friendliai)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [Galadriel](https://docs.litellm.ai/docs/providers/galadriel)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [GradientAI](https://docs.litellm.ai/docs/providers/gradient_ai)                              | ✅                                                       | ✅                                                                               |                                                                                   |                                                                                  |                                                                               |                                                                         |
-| [Novita AI](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link)                     | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [Featherless AI](https://docs.litellm.ai/docs/providers/featherless_ai)                              | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 |                                                                               |                                                                         |
-| [Nebius AI Studio](https://docs.litellm.ai/docs/providers/nebius)                             | ✅                                                       | ✅                                                                               | ✅                                                                                   | ✅                                                                                 | ✅                                                                             |                                                                         |
-| [Heroku](https://docs.litellm.ai/docs/providers/heroku)                             | ✅                                                       | ✅                                                                               |                                                                                    |                                                                                  |                                                                              |                                                                         |
-| [OVHCloud AI Endpoints](https://docs.litellm.ai/docs/providers/ovhcloud)                             | ✅                                                       | ✅                                                                               |                                                                                    |                                                                                  |                                                                              |                                                                         |
+</details>
+
+---
+
+## How to use LiteLLM
+
+You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
+
+<table style={{width: '100%', tableLayout: 'fixed'}}>
+<thead>
+<tr>
+<th style={{width: '14%'}}></th>
+<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
+<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td style={{width: '14%'}}><strong>Use Case</strong></td>
+<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
+<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
+</tr>
+<tr>
+<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
+<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
+<td style={{width: '43%'}}>Developers building LLM projects</td>
+</tr>
+<tr>
+<td style={{width: '14%'}}><strong>Key Features</strong></td>
+<td style={{width: '43%'}}>Centralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and management</td>
+<td style={{width: '43%'}}>Direct Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a>, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
+</tr>
+</tbody>
+</table>
+
+LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
+
+[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy) <br>
+[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
+
+**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
+
+Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
+
+## OSS Adopters 
+
+<table>
+  <tr>
+    <td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
+    <td><img height="60" alt="image" src="https://github.com/user-attachments/assets/436fca71-988b-40bb-b5fe-8450c80fdbd0" /></td>
+    <td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
+    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
+    <td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
+    <td><h2>Netflix</h2></td>
+    <td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
+  </tr>
+</table>
+
+## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
+
+| Provider                                                                            | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
+|-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
+| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ |  |  |  |  |  |  |  |  |  |
+| [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |
+| [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Amazon Nova](https://docs.litellm.ai/docs/providers/amazon_nova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |  |
+| [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |  |
+| [Anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [AssemblyAI (`assemblyai`)](https://docs.litellm.ai/docs/pass_through/assembly_ai) | ✅ | ✅ | ✅ |  |  | ✅ |  |  |  |  |
+| [Auto Router (`auto_router`)](https://docs.litellm.ai/docs/proxy/auto_routing) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [AWS - Bedrock (`bedrock`)](https://docs.litellm.ai/docs/providers/bedrock) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |
+| [AWS - Sagemaker (`sagemaker`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
+| [Azure (`azure`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |
+| [Azure AI (`azure_ai`)](https://docs.litellm.ai/docs/providers/azure_ai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |
+| [Azure Text (`azure_text`)](https://docs.litellm.ai/docs/providers/azure) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ |  |
+| [Baseten (`baseten`)](https://docs.litellm.ai/docs/providers/baseten) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Cohere (`cohere`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |
+| [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
+| [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ |  |
+| [Dashscope (`dashscope`)](https://docs.litellm.ai/docs/providers/dashscope) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Databricks (`databricks`)](https://docs.litellm.ai/docs/providers/databricks) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [DataRobot (`datarobot`)](https://docs.litellm.ai/docs/providers/datarobot) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ |  |  | ✅ |  |  |  |  |
+| [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ |  |  |  |
+| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |
+| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [FriendliAI (`friendliai`)](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Galadriel (`galadriel`)](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
+| [GitHub Models (`github`)](https://docs.litellm.ai/docs/providers/github) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Google - PaLM](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Google - Vertex AI (`vertex_ai`)](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |
+| [Google AI Studio - Gemini (`gemini`)](https://docs.litellm.ai/docs/providers/gemini) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [GradientAI (`gradient_ai`)](https://docs.litellm.ai/docs/providers/gradient_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Groq AI (`groq`)](https://docs.litellm.ai/docs/providers/groq) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Heroku (`heroku`)](https://docs.litellm.ai/docs/providers/heroku) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Hosted VLLM (`hosted_vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Huggingface (`huggingface`)](https://docs.litellm.ai/docs/providers/huggingface) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |
+| [Hyperbolic (`hyperbolic`)](https://docs.litellm.ai/docs/providers/hyperbolic) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [IBM - Watsonx.ai (`watsonx`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
+| [Infinity (`infinity`)](https://docs.litellm.ai/docs/providers/infinity) |  |  |  | ✅ |  |  |  |  |  |  |
+| [Jina AI (`jina_ai`)](https://docs.litellm.ai/docs/providers/jina_ai) |  |  |  | ✅ |  |  |  |  |  |  |
+| [Lambda AI (`lambda_ai`)](https://docs.litellm.ai/docs/providers/lambda_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Lemonade (`lemonade`)](https://docs.litellm.ai/docs/providers/lemonade) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [LiteLLM Proxy (`litellm_proxy`)](https://docs.litellm.ai/docs/providers/litellm_proxy) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |
+| [Llamafile (`llamafile`)](https://docs.litellm.ai/docs/providers/llamafile) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [LM Studio (`lm_studio`)](https://docs.litellm.ai/docs/providers/lm_studio) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Maritalk (`maritalk`)](https://docs.litellm.ai/docs/providers/maritalk) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Meta - Llama API (`meta_llama`)](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Mistral AI API (`mistral`)](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
+| [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
+| [NLP Cloud (`nlp_cloud`)](https://docs.litellm.ai/docs/providers/nlp_cloud) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Novita AI (`novita`)](https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Nscale (`nscale`)](https://docs.litellm.ai/docs/providers/nscale) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Nvidia NIM (`nvidia_nim`)](https://docs.litellm.ai/docs/providers/nvidia_nim) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [OCI (`oci`)](https://docs.litellm.ai/docs/providers/oci) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Ollama (`ollama`)](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
+| [Ollama Chat (`ollama_chat`)](https://docs.litellm.ai/docs/providers/ollama) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Oobabooga (`oobabooga`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ |  |
+| [OpenAI (`openai`)](https://docs.litellm.ai/docs/providers/openai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |
+| [OpenAI-like (`openai_like`)](https://docs.litellm.ai/docs/providers/openai_compatible) |  |  |  | ✅ |  |  |  |  |  |  |
+| [OpenRouter (`openrouter`)](https://docs.litellm.ai/docs/providers/openrouter) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [OVHCloud AI Endpoints (`ovhcloud`)](https://docs.litellm.ai/docs/providers/ovhcloud) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Perplexity AI (`perplexity`)](https://docs.litellm.ai/docs/providers/perplexity) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Petals (`petals`)](https://docs.litellm.ai/docs/providers/petals) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Predibase (`predibase`)](https://docs.litellm.ai/docs/providers/predibase) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Recraft (`recraft`)](https://docs.litellm.ai/docs/providers/recraft) |  |  |  |  | ✅ |  |  |  |  |  |
+| [Replicate (`replicate`)](https://docs.litellm.ai/docs/providers/replicate) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ | ✅ | ✅ |  |
+| [Together AI (`together_ai`)](https://docs.litellm.ai/docs/providers/togetherai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Topaz (`topaz`)](https://docs.litellm.ai/docs/providers/topaz) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Triton (`triton`)](https://docs.litellm.ai/docs/providers/triton-inference-server) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [V0 (`v0`)](https://docs.litellm.ai/docs/providers/v0) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Volcengine (`volcengine`)](https://docs.litellm.ai/docs/providers/volcano) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Voyage AI (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) |  |  |  | ✅ |  |  |  |  |  |  |
+| [WandB Inference (`wandb`)](https://docs.litellm.ai/docs/providers/wandb_inference) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Xinference (`xinference`)](https://docs.litellm.ai/docs/providers/xinference) |  |  |  | ✅ |  |  |  |  |  |  |
 
 [**Read the Docs**](https://docs.litellm.ai/docs/)
 
-## Contributing
+## Run in Developer mode
+### Services
+1. Setup .env file in root
+2. Run dependant services `docker-compose up db prometheus`
 
-Interested in contributing? Contributions to LiteLLM Python SDK, Proxy Server, and LLM integrations are both accepted and highly encouraged!
+### Backend
+1. (In root) create virtual environment `python -m venv .venv`
+2. Activate virtual environment `source .venv/bin/activate`
+3. Install dependencies `pip install -e ".[all]"`
+4. `pip install prisma`
+5. `prisma generate`
+6. Start proxy backend `python litellm/proxy/proxy_cli.py`
 
-**Quick start:** `git clone` → `make install-dev` → `make format` → `make lint` → `make test-unit`
-
-See our comprehensive [Contributing Guide (CONTRIBUTING.md)](CONTRIBUTING.md) for detailed instructions.
+### Frontend
+1. Navigate to `ui/litellm-dashboard`
+2. Install dependencies `npm install`
+3. Run `npm run dev` to start the dashboard
 
 # Enterprise
 For companies that need better security, user management and professional support
 
-[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+[Talk to founders](https://enterprise.litellm.ai/demo)
 
 This covers:
 - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
@@ -411,7 +453,6 @@ All these checks must pass before your PR can be merged.
 - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version)
 - [Community Discord 💭](https://discord.gg/wuPM9dRgDw)
 - [Community Slack 💭](https://www.litellm.ai/support)
-- Our numbers 📞 +1 (770) 8783-106 / ‭+1 (412) 618-6238‬
 - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
 
 # Why did we build this
@@ -433,19 +474,3 @@ All these checks must pass before your PR can be merged.
   <img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
 </a>
 
-
-## Run in Developer mode
-### Services
-1. Setup .env file in root
-2. Run dependant services `docker-compose up db prometheus`
-
-### Backend
-1. (In root) create virtual environment `python -m venv .venv`
-2. Activate virtual environment `source .venv/bin/activate`
-3. Install dependencies `pip install -e ".[all]"`
-4. Start proxy backend `python3 /path/to/litellm/proxy_cli.py`
-
-### Frontend
-1. Navigate to `ui/litellm-dashboard`
-2. Install dependencies `npm install`
-3. Run `npm run dev` to start the dashboard

ci_cd/.grype.yaml

@@ -0,0 +1,36 @@
+ignore:
+  - vulnerability: CVE-2026-22184
+    reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
+  # Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable
+  - vulnerability: CVE-2025-55130
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-59465
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-55131
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-59466
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2026-21637
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-55132
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: GHSA-hx9q-6w63-j58v
+    reason: orjson dumps recursion; allowlisted
+  - vulnerability: GHSA-73rr-hh4g-fpgx
+    reason: diff npm transitive dep; override in package.json, allowlisted
+  - vulnerability: CVE-2026-0865
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-15282
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2026-0672
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-15366
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-15367
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-11468
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-12781
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2026-1299
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet

ci_cd/TEST_KEY_PATTERNS.md

@@ -0,0 +1,40 @@
+# Test Key Patterns Standard
+
+Standard patterns for test/mock keys and credentials in the LiteLLM codebase to avoid triggering secret detection.
+
+## How GitGuardian Works
+
+GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
+- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
+- **High entropy** values (realistic-looking secrets) trigger detection
+- **Context-aware** detection understands code syntax like `os.environ["KEY"]`
+
+## Recommended Test Key Patterns
+
+### Option 1: Low Entropy Values (Simplest)
+These won't trigger GitGuardian's ML detector:
+
+```python
+api_key = "sk-1234"
+api_key = "sk-12345"
+database_password = "postgres"
+token = "test123"
+```
+
+### Option 2: High Entropy with Test Prefixes
+If you need realistic-looking test keys with high entropy, use these prefixes:
+
+```python
+api_key = "sk-test-abc123def456ghi789..."  # OpenAI-style test key
+api_key = "sk-mock-1234567890abcdef1234..."  # Mock key
+api_key = "sk-fake-xyz789uvw456rst123..."  # Fake key
+token = "test-api-key-with-high-entropy"
+```
+
+## Configured Ignore Patterns
+
+These patterns are in `.gitguardian.yaml` for high-entropy test keys:
+- `sk-test-*` - OpenAI-style test keys
+- `sk-mock-*` - Mock API keys  
+- `sk-fake-*` - Fake API keys
+- `test-api-key` - Generic test tokens

ci_cd/security_scans.sh

@@ -10,13 +10,13 @@ echo "Starting security scans for LiteLLM..."
 # Function to install Trivy and required tools
 install_trivy() {
     echo "Installing Trivy and required tools..."
+    TRIVY_VERSION="0.35.0"
     sudo apt-get update
-    sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
-    wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
-    echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
-    sudo apt-get update
-    sudo apt-get install trivy
-    echo "Trivy and required tools installed successfully"
+    sudo apt-get install -y wget jq curl bsdmainutils
+    wget -qO trivy.deb "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.deb"
+    sudo dpkg -i trivy.deb
+    rm trivy.deb
+    echo "Trivy ${TRIVY_VERSION} installed successfully"
 }
 
 # Function to install Grype
@@ -26,15 +26,65 @@ install_grype() {
     echo "Grype installed successfully"
 }
 
+# Function to install ggshield
+install_ggshield() {
+    echo "Installing ggshield..."
+    pip3 install --upgrade pip
+    pip3 install ggshield
+    echo "ggshield installed successfully"
+}
+
+# # Function to run secret detection scans
+# run_secret_detection() {
+#     echo "Running secret detection scans..."
+    
+#     if ! command -v ggshield &> /dev/null; then
+#         install_ggshield
+#     fi
+    
+#     # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
+#     if [ -z "$GITGUARDIAN_API_KEY" ]; then
+#         echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
+#         echo "ggshield requires a GitGuardian API key to scan for secrets."
+#         echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
+#         exit 1
+#     fi
+    
+#     echo "Scanning codebase for secrets..."
+#     echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
+#     echo "ggshield will automatically handle rate limits and retry as needed."
+#     echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
+    
+#     # Use --recursive for directory scanning and auto-confirm if prompted
+#     # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
+#     # GITGUARDIAN_API_KEY environment variable will be used for authentication
+#     echo y | ggshield secret scan path . --recursive || {
+#         echo ""
+#         echo "=========================================="
+#         echo "ERROR: Secret Detection Failed"
+#         echo "=========================================="
+#         echo "ggshield has detected secrets in the codebase."
+#         echo "Please review discovered secrets above, revoke any actively used secrets"
+#         echo "from underlying systems and make changes to inject secrets dynamically at runtime."
+#         echo ""
+#         echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
+#         echo "=========================================="
+#         echo ""
+#         exit 1
+#     }
+    
+#     echo "Secret detection scans completed successfully"
+# }
+
 # Function to run Trivy scans
 run_trivy_scans() {
     echo "Running Trivy scans..."
     
     echo "Scanning LiteLLM Docs..."
-    trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
+    trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
     
     echo "Scanning LiteLLM UI..."
-    trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
+    trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
     
     echo "Trivy scans completed successfully"
 }
@@ -50,13 +100,13 @@ run_grype_scans() {
     
     # Build and scan Dockerfile.database
     echo "Building and scanning Dockerfile.database..."
-    docker build -t litellm-database:latest -f ./docker/Dockerfile.database .
-    grype litellm-database:latest --fail-on critical
+    docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
+    grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical
     
     # Build and scan main Dockerfile
     echo "Building and scanning main Dockerfile..."
-    docker build -t litellm:latest .
-    grype litellm:latest --fail-on critical
+    docker build --no-cache -t litellm:latest .
+    grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical
     
     # Restore original .dockerignore
     echo "Restoring original .dockerignore..."
@@ -66,18 +116,122 @@ run_grype_scans() {
     echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
     echo "Using locally built image: litellm:latest"
     
-    # Run grype scan and check for vulnerabilities with CVSS >= 4.0
+    # Allowlist of CVEs to be ignored in failure threshold/reporting
+    # - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
+    # - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869
+    # - GHSA-5j98-mcp5-4vw2: glob CLI command injection via -c/--cmd; glob CLI is not used in the litellm runtime image,
+    #   and the vulnerable versions are pulled in only via OS-level/node tooling outside of our application code
+    ALLOWED_CVES=(
+        "CVE-2025-8869"
+        "GHSA-4xh5-x5gv-qwph"
+        "CVE-2025-8291" # no fix available as of Oct 11, 2025
+        "GHSA-5j98-mcp5-4vw2"
+        "CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
+        "CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
+        "CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
+        "CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
+        "CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
+        "CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
+        "CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
+        "CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
+        "CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
+        "CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
+        "GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
+        "GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
+        "GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
+        "GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
+        "CVE-2025-59465" # Node only used for Admin UI build/prisma
+        "CVE-2025-55131" # Node only used for Admin UI build/prisma
+        "CVE-2025-59466" # Node only used for Admin UI build/prisma
+        "CVE-2025-55130" # Node only used for Admin UI build/prisma
+        "CVE-2025-59467" # Node only used for Admin UI build/prisma
+        "CVE-2026-21637" # Node only used for Admin UI build/prisma
+        "CVE-2025-55132" # Node only used for Admin UI build/prisma
+        "GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted
+        "CVE-2025-15281" # No fix available yet
+        "CVE-2026-0865" # No fix available yet
+        "CVE-2025-15282" # No fix available yet
+        "CVE-2026-0672" # No fix available yet
+        "CVE-2025-15366" # No fix available yet
+        "CVE-2025-15367" # No fix available yet
+        "CVE-2025-12781" # No fix available yet
+        "CVE-2025-11468" # No fix available yet
+        "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
+        "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
+        "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
+        "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
+        "CVE-2026-25639" # axios DoS via __proto__ in mergeConfig - transitive dev dep via @neondatabase/api-client, not imported in application code
+        "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
+        "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
+        "CVE-2026-2673" # OpenSSL 3.6.1 TLS 1.3 key exchange group negotiation issue - no fix available yet
+        "CVE-2026-3644" # Python 3.13 vulnerability - no fix available in base image
+        "CVE-2026-4224" # Python 3.13 Expat parser stack overflow in ElementDeclHandler - no fix available in base image
+    )
+
+    # Build JSON array of allowlisted CVE IDs for jq
+    ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
+
     echo "Checking for vulnerabilities with CVSS score >= 4.0..."
-    HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq -r '.matches[] | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) | .vulnerability.id' | wc -l)
+    echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
+    echo ""
+    
+    # Show all high-severity vulnerabilities for transparency
+    TOTAL_HIGH_SEVERITY=$(grype litellm:latest -o json | jq -r '
+        .matches[]
+        | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+        | .vulnerability.id' | wc -l)
+    
+    if [ "$TOTAL_HIGH_SEVERITY" -gt 0 ]; then
+        echo "Total vulnerabilities found with CVSS >= 4.0: $TOTAL_HIGH_SEVERITY"
+        echo ""
+        echo "All high-severity vulnerabilities (including allowlisted):"
+        grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
+        ["Package", "Version", "Vulnerability ID", "CVSS Score", "Allowlisted"],
+        (.matches[]
+          | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+          | [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, (if (.vulnerability.id as $id | $allow | index($id)) then "YES" else "NO" end)])
+        | @tsv' | column -t -s $'\t'
+        echo ""
+    fi
+
+    HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
+        .matches[]
+        | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+        | select((.vulnerability.id as $id | $allow | index($id) | not))
+        | .vulnerability.id' | wc -l)
     
     if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
-        echo "ERROR: Found $HIGH_SEVERITY_COUNT vulnerabilities with CVSS score >= 4.0 in litellm:latest"
+        echo ""
+        echo "=========================================="
+        echo "ERROR: Security Scan Failed"
+        echo "=========================================="
+        echo "Found $HIGH_SEVERITY_COUNT non-allowlisted vulnerabilities with CVSS score >= 4.0 in litellm:latest"
+        echo ""
+        echo "These vulnerabilities are NOT in the allowlist and must be addressed."
+        echo "Current allowlisted CVEs: ${ALLOWED_CVES[*]}"
+        echo ""
         echo "Detailed vulnerability report:"
-        grype litellm:latest -o json | jq -r '
+        echo ""
+        grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
         ["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
-        (.matches[] | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) |
-        [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description]) |
-        @tsv' | column -t -s $'\t'
+        (.matches[]
+          | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
+          | select((.vulnerability.id as $id | $allow | index($id) | not))
+          | [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
+        | @tsv' | column -t -s $'\t'
+        echo ""
+        echo "=========================================="
+        echo "Action Required:"
+        echo "=========================================="
+        echo "1. If a fix is available, update the package to the fixed version"
+        echo "2. If the vulnerability is not applicable or has no fix:"
+        echo "   - Add the CVE/GHSA ID to ALLOWED_CVES array in ci_cd/security_scans.sh"
+        echo "   - Add a comment explaining why it's safe to ignore"
+        echo ""
+        echo "Note: Some vulnerabilities may have multiple IDs (CVE-XXXX and GHSA-XXXX)."
+        echo "Add all relevant IDs to the allowlist if they refer to the same issue."
+        echo "=========================================="
+        echo ""
         exit 1
     else
         echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"
@@ -92,6 +246,9 @@ main() {
     install_trivy
     install_grype
     
+    # echo "Running secret detection scans..."
+    # run_secret_detection
+    
     echo "Running filesystem vulnerability scans..."
     run_trivy_scans
     

cookbook/LiteLLM_HuggingFace.ipynb

@@ -131,7 +131,7 @@
         "            {\n",
         "                \"type\": \"image_url\",\n",
         "                \"image_url\": {\n",
-        "                    \"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\",\n",
+        "                    \"url\": \"https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png\",\n",
         "                },\n",
         "            },\n",
         "        ],\n",

cookbook/LiteLLM_PromptLayer.ipynb

@@ -39,7 +39,7 @@
         "import os\n",
         "os.environ['OPENAI_API_KEY'] = \"\"\n",
         "os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
-        "os.environ['PROMPTLAYER_API_KEY'] = \"pl_4ea2bb00a4dca1b8a70cebf2e9e11564\"\n",
+        "os.environ['PROMPTLAYER_API_KEY'] = \"test-promptlayer-key-123\"\n",
         "\n",
         "# Set Promptlayer as a success callback\n",
         "litellm.success_callback =['promptlayer']\n",

cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb

@@ -1,21 +1,10 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "kccfk0mHZ4Ad"
+      },
       "source": [
         "# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n",
         "\n",
@@ -32,29 +21,26 @@
         "To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
         "\n",
         "To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n"
-      ],
-      "metadata": {
-        "id": "kccfk0mHZ4Ad"
-      }
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "nmSClzCPaGH6"
+      },
       "source": [
         "## /chat/completion\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "nmSClzCPaGH6"
-      }
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### OpenAI Python SDK"
-      ],
       "metadata": {
         "id": "_vqcjwOVaKpO"
-      }
+      },
+      "source": [
+        "### OpenAI Python SDK"
+      ]
     },
     {
       "cell_type": "code",
@@ -94,15 +80,20 @@
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "## Function Calling"
-      ],
       "metadata": {
         "id": "AqkyKk9Scxgj"
-      }
+      },
+      "source": [
+        "## Function Calling"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wDg10VqLczE1"
+      },
+      "outputs": [],
       "source": [
         "from openai import OpenAI\n",
         "client = OpenAI(\n",
@@ -139,24 +130,24 @@
         ")\n",
         "\n",
         "print(completion)\n"
-      ],
-      "metadata": {
-        "id": "wDg10VqLczE1"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Azure OpenAI Python SDK"
-      ],
       "metadata": {
         "id": "YYoxLloSaNWW"
-      }
+      },
+      "source": [
+        "### Azure OpenAI Python SDK"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yA1XcgowaSRy"
+      },
+      "outputs": [],
       "source": [
         "import openai\n",
         "client = openai.AzureOpenAI(\n",
@@ -184,24 +175,24 @@
         ")\n",
         "\n",
         "print(response)"
-      ],
-      "metadata": {
-        "id": "yA1XcgowaSRy"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Langchain Python"
-      ],
       "metadata": {
         "id": "yl9qhDvnaTpL"
-      }
+      },
+      "source": [
+        "### Langchain Python"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5MUZgSquaW5t"
+      },
+      "outputs": [],
       "source": [
         "from langchain.chat_models import ChatOpenAI\n",
         "from langchain.prompts.chat import (\n",
@@ -239,24 +230,22 @@
         "response = chat(messages)\n",
         "\n",
         "print(response)"
-      ],
-      "metadata": {
-        "id": "5MUZgSquaW5t"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Curl"
-      ],
       "metadata": {
         "id": "B9eMgnULbRaz"
-      }
+      },
+      "source": [
+        "### Curl"
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "VWCCk5PFcmhS"
+      },
       "source": [
         "\n",
         "\n",
@@ -280,22 +269,24 @@
         "}'\n",
         "```\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "VWCCk5PFcmhS"
-      }
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### LlamaIndex"
-      ],
       "metadata": {
         "id": "drBAm2e1b6xe"
-      }
+      },
+      "source": [
+        "### LlamaIndex"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d0bZcv8fb9mL"
+      },
+      "outputs": [],
       "source": [
         "import os, dotenv\n",
         "\n",
@@ -326,24 +317,24 @@
         "query_engine = index.as_query_engine()\n",
         "response = query_engine.query(\"What did the author do growing up?\")\n",
         "print(response)\n"
-      ],
-      "metadata": {
-        "id": "d0bZcv8fb9mL"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Langchain JS"
-      ],
       "metadata": {
         "id": "xypvNdHnb-Yy"
-      }
+      },
+      "source": [
+        "### Langchain JS"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R55mK2vCcBN2"
+      },
+      "outputs": [],
       "source": [
         "import { ChatOpenAI } from \"@langchain/openai\";\n",
         "\n",
@@ -359,24 +350,24 @@
         "const message = await model.invoke(\"Hi there!\");\n",
         "\n",
         "console.log(message);\n"
-      ],
-      "metadata": {
-        "id": "R55mK2vCcBN2"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### OpenAI JS"
-      ],
       "metadata": {
         "id": "nC4bLifCcCiW"
-      }
+      },
+      "source": [
+        "### OpenAI JS"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MICH8kIMcFpg"
+      },
+      "outputs": [],
       "source": [
         "const { OpenAI } = require('openai');\n",
         "\n",
@@ -398,24 +389,24 @@
         "}\n",
         "\n",
         "main();\n"
-      ],
-      "metadata": {
-        "id": "MICH8kIMcFpg"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Anthropic SDK"
-      ],
       "metadata": {
         "id": "D1Q07pEAcGTb"
-      }
+      },
+      "source": [
+        "### Anthropic SDK"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qBjFcAvgcI3t"
+      },
+      "outputs": [],
       "source": [
         "import os\n",
         "\n",
@@ -423,7 +414,7 @@
         "\n",
         "client = Anthropic(\n",
         "    base_url=\"http://localhost:4000\", # proxy endpoint\n",
-        "    api_key=\"sk-s4xN1IiLTCytwtZFJaYQrA\", # litellm proxy virtual key\n",
+        "    api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n",
         ")\n",
         "\n",
         "message = client.messages.create(\n",
@@ -437,33 +428,33 @@
         "    model=\"claude-3-opus-20240229\",\n",
         ")\n",
         "print(message.content)"
-      ],
-      "metadata": {
-        "id": "qBjFcAvgcI3t"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "## /embeddings"
-      ],
       "metadata": {
         "id": "dFAR4AJGcONI"
-      }
+      },
+      "source": [
+        "## /embeddings"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### OpenAI Python SDK"
-      ],
       "metadata": {
         "id": "lgNoM281cRzR"
-      }
+      },
+      "source": [
+        "### OpenAI Python SDK"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NY3DJhPfcQhA"
+      },
+      "outputs": [],
       "source": [
         "import openai\n",
         "from openai import OpenAI\n",
@@ -478,24 +469,24 @@
         ")\n",
         "\n",
         "print(response)\n"
-      ],
-      "metadata": {
-        "id": "NY3DJhPfcQhA"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Langchain Embeddings"
-      ],
       "metadata": {
         "id": "hmbg-DW6cUZs"
-      }
+      },
+      "source": [
+        "### Langchain Embeddings"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lX2S8Nl1cWVP"
+      },
+      "outputs": [],
       "source": [
         "from langchain.embeddings import OpenAIEmbeddings\n",
         "\n",
@@ -526,24 +517,22 @@
         "\n",
         "print(f\"TITAN EMBEDDINGS\")\n",
         "print(query_result[:5])"
-      ],
-      "metadata": {
-        "id": "lX2S8Nl1cWVP"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Curl Request"
-      ],
       "metadata": {
         "id": "oqGbWBCQcYfd"
-      }
+      },
+      "source": [
+        "### Curl Request"
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "7rkIMV9LcdwQ"
+      },
       "source": [
         "\n",
         "\n",
@@ -556,10 +545,21 @@
         "  }'\n",
         "```\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "7rkIMV9LcdwQ"
-      }
+      ]
     }
-  ]
-}
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md

@@ -0,0 +1,295 @@
+# Claude Code with LiteLLM Quickstart
+
+This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code.
+
+> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
+
+## Video Walkthrough
+
+Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56
+
+## Prerequisites
+
+- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
+- API keys for your chosen providers
+
+## Installation
+
+First, install LiteLLM with proxy support:
+
+```bash
+pip install 'litellm[proxy]'
+```
+
+## Step 1: Setup config.yaml
+
+Create a secure configuration using environment variables:
+
+```yaml
+model_list:
+  # Claude models
+  - model_name: claude-3-5-sonnet-20241022    
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+  
+  - model_name: claude-3-5-haiku-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-haiku-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+Set your environment variables:
+
+```bash
+export ANTHROPIC_API_KEY="your-anthropic-api-key"
+export LITELLM_MASTER_KEY="sk-1234567890"  # Generate a secure key
+```
+
+## Step 2: Start Proxy
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+## Step 3: Verify Setup
+
+Test that your proxy is working correctly:
+
+```bash
+curl -X POST http://0.0.0.0:4000/v1/messages \
+-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+-H "Content-Type: application/json" \
+-d '{
+    "model": "claude-3-5-sonnet-20241022",
+    "max_tokens": 1000,
+    "messages": [{"role": "user", "content": "What is the capital of France?"}]
+}'
+```
+
+## Step 4: Configure Claude Code
+
+### Method 1: Unified Endpoint (Recommended)
+
+Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here:
+
+```bash
+export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
+export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
+```
+
+> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI.
+
+### Method 2: Provider-specific Pass-through Endpoint
+
+Alternatively, use the Anthropic pass-through endpoint:
+
+```bash
+export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
+export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
+```
+
+## Step 5: Use Claude Code
+
+### Choosing Your Model
+
+You have two options for specifying which model Claude Code uses:
+
+#### Option 1: Command Line / Session Model Selection
+
+Specify the model directly when starting Claude Code or during a session:
+
+```bash
+# Specify model at startup
+claude --model claude-3-5-sonnet-20241022
+
+# Or change model during a session
+/model claude-3-5-haiku-20241022
+```
+
+This method uses the exact model you specify.
+
+#### Option 2: Environment Variables
+
+Configure default models using environment variables:
+
+```bash
+# Tell Claude Code which models to use by default
+export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
+export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
+export ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-3-5-20240229
+
+claude  # Will use the models specified above
+```
+
+**Note:** Claude Code may cache the model from a previous session. If environment variables don't take effect, use Option 1 to explicitly set the model.
+
+**Important:** The `model_name` in your LiteLLM config must match what Claude Code requests (either from env vars or command line).
+
+### Using 1M Context Window
+
+Claude Code supports extended context (1 million tokens) using the `[1m]` suffix with Claude 4+ models:
+
+```bash
+# Use Sonnet 4.5 with 1M context (requires quotes for shell)
+claude --model 'claude-sonnet-4-5-20250929[1m]'
+
+# Inside a Claude Code session (no quotes needed)
+/model claude-sonnet-4-5-20250929[1m]
+```
+
+**Important:** When using `--model` with `[1m]` in the shell, you must use quotes to prevent the shell from interpreting the brackets.
+
+Alternatively, set as default with environment variables:
+
+```bash
+export ANTHROPIC_DEFAULT_SONNET_MODEL='claude-sonnet-4-5-20250929[1m]'
+claude
+```
+
+**How it works:**
+- Claude Code strips the `[1m]` suffix before sending to LiteLLM
+- Claude Code automatically adds the header `anthropic-beta: context-1m-2025-08-07`
+- Your LiteLLM config should **NOT** include `[1m]` in model names
+
+**Verify 1M context is active:**
+```bash
+/context
+# Should show: 21k/1000k tokens (2%)
+```
+
+**Pricing:** Models using 1M context have different pricing. Input tokens above 200k are charged at a higher rate.
+
+## Troubleshooting
+
+Common issues and solutions:
+
+**Claude Code not connecting:**
+- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
+- Check that `ANTHROPIC_BASE_URL` is set correctly
+- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
+
+**Authentication errors:**
+- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
+- Check that your API keys are valid and have sufficient credits
+- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
+
+**Model not found:**
+- Check what model Claude Code is requesting in LiteLLM logs
+- Ensure your `config.yaml` has a matching `model_name` entry
+- If using environment variables, verify they're set: `echo $ANTHROPIC_DEFAULT_SONNET_MODEL`
+
+**1M context not working (showing 200k instead of 1000k):**
+- Verify you're using the `[1m]` suffix: `/model your-model-name[1m]`
+- Check LiteLLM logs for the header `context-1m-2025-08-07` in the request
+- Ensure your model supports 1M context (only certain Claude models do)
+- Your LiteLLM config should **NOT** include `[1m]` in the `model_name`
+
+## Using Multiple Models and Providers
+
+You can configure LiteLLM to route to any supported provider. Here's an example with multiple providers:
+
+```yaml
+model_list:
+  # OpenAI models
+  - model_name: codex-mini
+    litellm_params:
+      model: openai/codex-mini
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: https://api.openai.com/v1
+
+  - model_name: o3-pro
+    litellm_params:
+      model: openai/o3-pro
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: https://api.openai.com/v1
+
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: https://api.openai.com/v1
+
+  # Anthropic models
+  - model_name: claude-3-5-sonnet-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: claude-3-5-haiku-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-haiku-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  # AWS Bedrock
+  - model_name: claude-bedrock
+    litellm_params:
+      model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-east-1
+
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+**Note:** The `model_name` can be anything you choose. Claude Code will request whatever model you specify (via env vars or command line), and LiteLLM will route to the `model` configured in `litellm_params`.
+
+Switch between models seamlessly:
+
+```bash
+# Use environment variables to set defaults
+export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
+export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
+
+# Or specify directly
+claude --model claude-3-5-sonnet-20241022  # Complex reasoning
+claude --model claude-3-5-haiku-20241022    # Fast responses
+claude --model claude-bedrock                # Bedrock deployment
+```
+
+## Default Models Used by Claude Code
+
+If you **don't** set environment variables, Claude Code uses these default model names:
+
+| Purpose | Default Model Name (v2.1.14) |
+|---------|------------------------------|
+| Main model | `claude-sonnet-4-5-20250929` |
+| Light tasks (subagents, summaries) | `claude-haiku-4-5-20251001` |
+| Planning mode | `claude-opus-4-5-20251101` |
+
+Your LiteLLM config should include these model names if you want Claude Code to work without setting environment variables:
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-5-20250929
+    litellm_params:
+      # Can be any provider - Anthropic, Bedrock, Vertex AI, etc.
+      model: anthropic/claude-sonnet-4-5-20250929
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: claude-haiku-4-5-20251001
+    litellm_params:
+      model: anthropic/claude-haiku-4-5-20251001
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: claude-opus-4-5-20251101
+    litellm_params:
+      model: anthropic/claude-opus-4-5-20251101
+      api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+**Warning:** These default model names may change with new Claude Code versions. Check LiteLLM proxy logs for "model not found" errors to identify what Claude Code is requesting.
+
+## Additional Resources
+
+- [LiteLLM Documentation](https://docs.litellm.ai/)
+- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
+- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)
+

cookbook/ai_coding_tool_guides/index.json

@@ -0,0 +1,134 @@
+[{
+    "title": "Claude Code Quickstart",
+    "description": "This is a quickstart guide to using Claude Code with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api",
+    "date": "2026-01-15",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM"
+    ]
+},
+{
+    "title": "Claude Code with MCPs",
+    "description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_mcp",
+    "date": "2026-01-15",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM",
+        "MCP"
+    ]
+}, 
+{
+    "title": "Claude Code with Non-Anthropic Models",
+    "description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM",
+        "OpenAI",
+        "Gemini"
+    ]
+},
+{
+    "title": "Cursor Quickstart",
+    "description": "This is a quickstart guide to using Cursor with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/cursor_integration",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Cursor",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "Github Copilot Quickstart",
+    "description": "This is a quickstart guide to using Github Copilot with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Github Copilot",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "LiteLLM Gemini CLI Quickstart",
+    "description": "This is a quickstart guide to using LiteLLM Gemini CLI.",
+    "url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Gemini CLI",
+        "Gemini",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "OpenAI Codex CLI Quickstart",
+    "description": "This is a quickstart guide to using OpenAI Codex CLI.",
+    "url": "https://docs.litellm.ai/docs/tutorials/openai_codex",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "OpenAI Codex CLI",
+        "OpenAI",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "OpenWebUI Quickstart",
+    "description": "This is a quickstart guide to using OpenWebUI with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/openweb_ui",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "OpenWebUI",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "AI Coding Tool Usage Tracking",
+    "description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding",
+    "date": "2026-01-17",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "Gemini CLI",
+        "OpenAI Codex",
+        "LiteLLM"
+    ]
+},
+{
+    "title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)",
+    "description": "This is a guide for using Web Search with Claude Code via LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch",
+    "date": "2026-01-17",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM",
+        "Web Search"
+    ]
+},
+{
+    "title": "Track Claude Code Usage per user via Custom Headers",
+    "description": "This is a guide for tracking claude code user usage by passing a customer ID header.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking",
+    "date": "2026-01-17",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM"
+    ]
+}]

cookbook/anthropic_agent_sdk/README.md

@@ -0,0 +1,144 @@
+# Claude Agent SDK with LiteLLM Gateway
+
+A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK.
+
+## Quick Start
+
+### 1. Install dependencies
+
+```bash
+pip install anthropic claude-agent-sdk litellm
+```
+
+### 2. Start LiteLLM proxy
+
+```bash
+# Simple start with Claude
+litellm --model claude-sonnet-4-20250514
+
+# Or with a config file
+litellm --config config.yaml
+```
+
+### 3. Run the chat
+
+**Basic Agent (no MCP):**
+
+```bash
+python main.py
+```
+
+**Agent with MCP (DeepWiki2 for research):**
+
+```bash
+python agent_with_mcp.py
+```
+
+If MCP connection fails, you can disable it:
+
+```bash
+USE_MCP=false python agent_with_mcp.py
+```
+
+That's it! You can now chat with the agent in your terminal.
+
+### Chat Commands
+
+While chatting, you can use these commands:
+- `models` - List all available models (fetched from your LiteLLM proxy)
+- `model` - Switch to a different model
+- `clear` - Start a new conversation
+- `quit` or `exit` - End the chat
+
+The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured.
+
+## Configuration
+
+Set these environment variables if needed:
+
+```bash
+export LITELLM_PROXY_URL="http://localhost:4000"
+export LITELLM_API_KEY="sk-1234"
+export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
+```
+
+Or just use the defaults - it'll connect to `http://localhost:4000` by default.
+
+## Files
+
+- `main.py` - Basic interactive agent without MCP
+- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2)
+- `common.py` - Shared utilities and functions
+- `config.example.yaml` - Example LiteLLM configuration
+- `requirements.txt` - Python dependencies
+
+## Example Config File
+
+If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`):
+
+```yaml
+model_list:
+  - model_name: bedrock-claude-sonnet-4
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+      aws_region_name: "us-east-1"
+```
+
+Then start LiteLLM with: `litellm --config config.yaml`
+
+## How It Works
+
+The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
+
+```python
+# Point to LiteLLM gateway (not Anthropic)
+os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
+os.environ["ANTHROPIC_API_KEY"] = "sk-1234"  # Your LiteLLM key
+
+# Use any model configured in LiteLLM
+options = ClaudeAgentOptions(
+    model="bedrock-claude-sonnet-4",  # or gpt-4, or anything else
+    system_prompt="You are a helpful assistant.",
+    max_turns=50,
+)
+```
+
+Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically.
+
+## Why Use This?
+
+- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc.
+- **Cost tracking**: LiteLLM tracks spending across all your agent conversations
+- **Rate limiting**: Set budgets and limits on your agent usage
+- **Load balancing**: Distribute requests across multiple API keys or regions
+- **Fallbacks**: Automatically retry with a different model if one fails
+
+## Troubleshooting
+
+**Connection errors?**
+- Make sure LiteLLM is running: `litellm --model your-model`
+- Check the URL is correct (default: `http://localhost:4000`)
+
+**Authentication errors?**
+- Verify your LiteLLM API key is correct
+- Make sure the model is configured in your LiteLLM setup
+
+**Model not found?**
+- Check the model name matches what's in your LiteLLM config
+- Run `litellm --model your-model` to test it works
+
+**Agent with MCP stuck or failing?**
+- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2`
+- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py`
+- Or use the basic agent: `python main.py`
+
+## Learn More
+
+- [LiteLLM Docs](https://docs.litellm.ai/)
+- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk)
+- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start)

cookbook/anthropic_agent_sdk/agent_with_mcp.py

@@ -0,0 +1,140 @@
+"""
+Interactive Claude Agent SDK CLI with MCP Support
+
+This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy,
+with MCP (Model Context Protocol) server integration for enhanced capabilities.
+"""
+
+import asyncio
+import os
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+from common import (
+    Config,
+    fetch_available_models,
+    setup_litellm_env,
+    print_header,
+    handle_model_list,
+    handle_model_switch,
+    stream_response,
+)
+
+
+async def interactive_chat_with_mcp():
+    """
+    Interactive CLI chat with the agent and MCP server
+    """
+    config = Config()
+    
+    # Configure Anthropic SDK to point to LiteLLM gateway
+    litellm_base_url = setup_litellm_env(config)
+    
+    # Fetch available models from proxy
+    available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
+    
+    current_model = config.LITELLM_MODEL
+    
+    # MCP server configuration
+    mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
+    use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
+    
+    if not use_mcp:
+        print("⚠️  MCP disabled via USE_MCP=false")
+    
+    print_header(litellm_base_url, current_model, has_mcp=use_mcp)
+    
+    while True:
+        # Configure agent options
+        if use_mcp:
+            try:
+                # Try with MCP server (HTTP transport)
+                # Using McpHttpServerConfig format from Agent SDK
+                options = ClaudeAgentOptions(
+                    system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.",
+                    model=current_model,
+                    max_turns=50,
+                    mcp_servers={
+                        "deepwiki2": {
+                            "type": "http",
+                            "url": mcp_server_url,
+                            "headers": {
+                                "Authorization": f"Bearer {config.LITELLM_API_KEY}"
+                            }
+                        }
+                    },
+                )
+            except Exception as e:
+                print(f"⚠️  Warning: Could not configure MCP server: {e}")
+                print("Continuing without MCP...\n")
+                use_mcp = False
+                options = ClaudeAgentOptions(
+                    system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+                    model=current_model,
+                    max_turns=50,
+                )
+        else:
+            # Without MCP
+            options = ClaudeAgentOptions(
+                system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+                model=current_model,
+                max_turns=50,
+            )
+        
+        # Create agent client
+        try:
+            async with ClaudeSDKClient(options=options) as client:
+                conversation_active = True
+                
+                while conversation_active:
+                    # Get user input
+                    try:
+                        user_input = input("\n👤 You: ").strip()
+                    except (EOFError, KeyboardInterrupt):
+                        print("\n\n👋 Goodbye!")
+                        return
+                    
+                    # Handle commands
+                    if user_input.lower() in ['quit', 'exit']:
+                        print("\n👋 Goodbye!")
+                        return
+                    
+                    if user_input.lower() == 'clear':
+                        print("\n🔄 Starting new conversation...\n")
+                        conversation_active = False
+                        continue
+                    
+                    if user_input.lower() == 'models':
+                        handle_model_list(available_models, current_model)
+                        continue
+                    
+                    if user_input.lower() == 'model':
+                        new_model, should_restart = handle_model_switch(available_models, current_model)
+                        if should_restart:
+                            current_model = new_model
+                            conversation_active = False
+                        continue
+                    
+                    if not user_input:
+                        continue
+                    
+                    # Stream response from agent
+                    await stream_response(client, user_input)
+        
+        except Exception as e:
+            print(f"\n❌ Error creating agent client: {e}")
+            print("This might be an MCP configuration issue. Try running without MCP:")
+            print("  USE_MCP=false python agent_with_mcp.py")
+            print("\nOr use the basic agent:")
+            print("  python main.py")
+            return
+
+
+def main():
+    """Run interactive chat with MCP"""
+    try:
+        asyncio.run(interactive_chat_with_mcp())
+    except KeyboardInterrupt:
+        print("\n\n👋 Goodbye!")
+
+
+if __name__ == "__main__":
+    main()

cookbook/anthropic_agent_sdk/common.py

@@ -0,0 +1,160 @@
+"""
+Common utilities for Claude Agent SDK examples
+"""
+
+import os
+import httpx
+
+
+class Config:
+    """Configuration for LiteLLM Gateway connection"""
+    
+    # LiteLLM proxy URL (default to local instance)
+    LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
+    
+    # LiteLLM API key (master key or virtual key)
+    LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
+    
+    # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
+    LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
+
+
+async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
+    """
+    Fetch available models from LiteLLM proxy /models endpoint
+    """
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(
+                f"{base_url}/models",
+                headers={"Authorization": f"Bearer {api_key}"},
+                timeout=10.0
+            )
+            response.raise_for_status()
+            data = response.json()
+            return [model["id"] for model in data.get("data", [])]
+    except Exception as e:
+        print(f"⚠️  Warning: Could not fetch models from proxy: {e}")
+        print("Using default model list...")
+        # Fallback to default models
+        return [
+            "bedrock-claude-sonnet-3.5",
+            "bedrock-claude-sonnet-4",
+            "bedrock-claude-sonnet-4.5",
+            "bedrock-claude-opus-4.5",
+            "bedrock-nova-premier",
+        ]
+
+
+def setup_litellm_env(config: Config):
+    """
+    Configure environment variables to point Agent SDK to LiteLLM
+    """
+    litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
+    os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
+    os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
+    return litellm_base_url
+
+
+def print_header(base_url: str, current_model: str, has_mcp: bool = False):
+    """
+    Print the chat header
+    """
+    mcp_indicator = " + MCP" if has_mcp else ""
+    print("=" * 70)
+    print(f"🤖 Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat")
+    print("=" * 70)
+    print(f"🚀 Connected to: {base_url}")
+    print(f"📦 Current model: {current_model}")
+    if has_mcp:
+        print("🔌 MCP: deepwiki2 enabled")
+    print("\nType your messages below. Commands:")
+    print("  - 'quit' or 'exit' to end the conversation")
+    print("  - 'clear' to start a new conversation")
+    print("  - 'model' to switch models")
+    print("  - 'models' to list available models")
+    print("=" * 70)
+    print()
+
+
+def handle_model_list(available_models: list[str], current_model: str):
+    """
+    Display available models
+    """
+    print("\n📋 Available models:")
+    for i, model in enumerate(available_models, 1):
+        marker = "✓" if model == current_model else " "
+        print(f"  {marker} {i}. {model}")
+
+
+def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
+    """
+    Handle model switching
+    
+    Returns:
+        tuple: (new_model, should_restart_conversation)
+    """
+    print("\n📋 Select a model:")
+    for i, model in enumerate(available_models, 1):
+        marker = "✓" if model == current_model else " "
+        print(f"  {marker} {i}. {model}")
+    
+    try:
+        choice = input("\nEnter number (or press Enter to cancel): ").strip()
+        if choice:
+            idx = int(choice) - 1
+            if 0 <= idx < len(available_models):
+                new_model = available_models[idx]
+                print(f"\n✅ Switched to: {new_model}")
+                print("🔄 Starting new conversation with new model...\n")
+                return new_model, True
+            else:
+                print("❌ Invalid choice")
+    except (ValueError, IndexError):
+        print("❌ Invalid input")
+    
+    return current_model, False
+
+
+async def stream_response(client, user_input: str):
+    """
+    Stream response from the agent
+    """
+    print("\n🤖 Assistant: ", end='', flush=True)
+    
+    try:
+        await client.query(user_input)
+        
+        # Show loading indicator
+        print("⏳ thinking...", end='', flush=True)
+        
+        # Stream the response
+        first_chunk = True
+        async for msg in client.receive_response():
+            # Clear loading indicator on first message
+            if first_chunk:
+                print("\r🤖 Assistant: ", end='', flush=True)
+                first_chunk = False
+            
+            # Handle different message types
+            if hasattr(msg, 'type'):
+                if msg.type == 'content_block_delta':
+                    # Streaming text delta
+                    if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
+                        print(msg.delta.text, end='', flush=True)
+                elif msg.type == 'content_block_start':
+                    # Start of content block
+                    if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
+                        print(msg.content_block.text, end='', flush=True)
+            
+            # Fallback to original content handling
+            if hasattr(msg, 'content'):
+                for content_block in msg.content:
+                    if hasattr(content_block, 'text'):
+                        print(content_block.text, end='', flush=True)
+        
+        print()  # New line after response
+        
+    except Exception as e:
+        print(f"\r\n❌ Error: {e}")
+        print("Please check your LiteLLM gateway is running and configured correctly.")

cookbook/anthropic_agent_sdk/config.example.yaml

@@ -0,0 +1,25 @@
+model_list:
+  - model_name: bedrock-claude-sonnet-3.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-opus-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-nova-premier
+    litellm_params:
+      model: "bedrock/amazon.nova-premier-v1:0"
+      aws_region_name: "us-east-1"

cookbook/anthropic_agent_sdk/main.py

@@ -0,0 +1,95 @@
+"""
+Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway
+
+This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy.
+LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.)
+through the Claude Agent SDK by pointing it to the LiteLLM gateway.
+"""
+
+import asyncio
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+from common import (
+    Config,
+    fetch_available_models,
+    setup_litellm_env,
+    print_header,
+    handle_model_list,
+    handle_model_switch,
+    stream_response,
+)
+
+
+async def interactive_chat():
+    """
+    Interactive CLI chat with the agent
+    """
+    config = Config()
+    
+    # Configure Anthropic SDK to point to LiteLLM gateway
+    litellm_base_url = setup_litellm_env(config)
+    
+    # Fetch available models from proxy
+    available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
+    
+    current_model = config.LITELLM_MODEL
+    
+    print_header(litellm_base_url, current_model)
+    
+    while True:
+        # Configure agent options for each conversation
+        options = ClaudeAgentOptions(
+            system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+            model=current_model,
+            max_turns=50,
+        )
+        
+        # Create agent client
+        async with ClaudeSDKClient(options=options) as client:
+            conversation_active = True
+            
+            while conversation_active:
+                # Get user input
+                try:
+                    user_input = input("\n👤 You: ").strip()
+                except (EOFError, KeyboardInterrupt):
+                    print("\n\n👋 Goodbye!")
+                    return
+                
+                # Handle commands
+                if user_input.lower() in ['quit', 'exit']:
+                    print("\n👋 Goodbye!")
+                    return
+                
+                if user_input.lower() == 'clear':
+                    print("\n🔄 Starting new conversation...\n")
+                    conversation_active = False
+                    continue
+                
+                if user_input.lower() == 'models':
+                    handle_model_list(available_models, current_model)
+                    continue
+                
+                if user_input.lower() == 'model':
+                    new_model, should_restart = handle_model_switch(available_models, current_model)
+                    if should_restart:
+                        current_model = new_model
+                        conversation_active = False
+                    continue
+                
+                if not user_input:
+                    continue
+                
+                # Stream response from agent
+                await stream_response(client, user_input)
+
+
+def main():
+    """Run interactive chat"""
+    try:
+        asyncio.run(interactive_chat())
+    except KeyboardInterrupt:
+        print("\n\n👋 Goodbye!")
+
+
+if __name__ == "__main__":
+    main()

cookbook/anthropic_agent_sdk/requirements.txt

@@ -0,0 +1,2 @@
+claude-agent-sdk
+httpx>=0.27.0