Merge branch 'main' into improve-llm-repeated-message-detection-performance

.circleci/requirements.txt

@@ -8,12 +8,14 @@ redis==5.2.1
 redisvl==0.4.1
 anthropic
 orjson==3.10.12 # fast /embedding responses
-pydantic==2.10.2
+pydantic==2.11.0
 google-cloud-aiplatform==1.43.0
 google-cloud-iam==2.19.1
 fastapi-sso==0.16.0 
 uvloop==0.21.0
-mcp==1.10.1    # for MCP server  
+mcp==1.25.0    # for MCP server  
 semantic_router==0.1.10 # for auto-routing with litellm
 fastuuid==0.12.0
-responses==0.25.7  # for proxy client tests
+responses==0.25.7  # for proxy client tests
+pytest-retry==1.6.3  # for automatic test retries
+litellm-proxy-extras  # for prisma migrations

.claude/settings.json

@@ -0,0 +1,36 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git show:*)",
+      "Bash(git worktree add:*)",
+      "Read(//Users/krrishdholakia/Documents/litellm/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)",
+      "Bash(python:*)",
+      "Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")",
+      "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)",
+      "Bash(poetry run pytest:*)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)",
+      "Bash(poetry run python:*)",
+      "Bash(poetry run pip:*)",
+      "Bash(git reset:*)",
+      "Bash(git cherry-pick:*)",
+      "Bash(git checkout:*)",
+      "Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)",
+      "Read(//Users/krrishdholakia/Documents/**)",
+      "Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)",
+      "Bash(ls:*)"
+    ],
+    "additionalDirectories": [
+      "/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails",
+      "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy",
+      "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth"
+    ]
+  }
+}

.dockerignore

@@ -48,7 +48,7 @@ dist/
 build/
 *.egg-info/
 .DS_Store
-node_modules/
+**/node_modules
 *.log
 .env
 .env.local

.gitguardian.yaml

@@ -0,0 +1,111 @@
+version: 2
+
+secret:
+  # Exclude files and paths by globbing
+  ignored_paths:
+    - "**/*.whl"
+    - "**/*.pyc"
+    - "**/__pycache__/**"
+    - "**/node_modules/**"
+    - "**/dist/**"
+    - "**/build/**"
+    - "**/.git/**"
+    - "**/venv/**"
+    - "**/.venv/**"
+    
+    # Large data/metadata files that don't need scanning
+    - "**/model_prices_and_context_window*.json"
+    - "**/*_metadata/*.txt"
+    - "**/tokenizers/*.json"
+    - "**/tokenizers/*"
+    - "miniconda.sh"
+    
+    # Build outputs and static assets
+    - "litellm/proxy/_experimental/out/**"
+    - "ui/litellm-dashboard/public/**"
+    - "**/swagger/*.js"
+    - "**/*.woff"
+    - "**/*.woff2"
+    - "**/*.avif"
+    - "**/*.webp"
+    
+    # Test data files
+    - "**/tests/**/data_map.txt"
+    - "tests/**/*.txt"
+    
+    # Documentation and other non-code files
+    - "docs/**"
+    - "**/*.md"
+    - "**/*.lock"
+    - "poetry.lock"
+    - "package-lock.json"
+
+  # Ignore security incidents with the SHA256 of the occurrence (false positives)
+  ignored_matches:
+    # === Current detected false positives (SHA-based) ===
+    
+    # gcs_pub_sub_body - folder name, not a password
+    - name: GCS pub/sub test folder name
+      match: 75f377c456eede69e5f6e47399ccee6016a2a93cc5dd11db09cc5b1359ae569a
+    
+    # os.environ/APORIA_API_KEY_1 - environment variable reference
+    - name: Environment variable reference APORIA_API_KEY_1
+      match: e2ddeb8b88eca97a402559a2be2117764e11c074d86159ef9ad2375dea188094
+    
+    # os.environ/APORIA_API_KEY_2 - environment variable reference
+    - name: Environment variable reference APORIA_API_KEY_2
+      match: 09aa39a29e050b86603aa55138af1ff08fb86a4582aa965c1bd0672e1575e052
+    
+    # oidc/circleci_v2/ - test authentication path, not a secret
+    - name: OIDC CircleCI test path
+      match: feb3475e1f89a65b7b7815ac4ec597e18a9ec1847742ad445c36ca617b536e15
+    
+    # text-davinci-003 - OpenAI model identifier, not a secret
+    - name: OpenAI model identifier text-davinci-003
+      match: c489000cf6c7600cee0eefb80ad0965f82921cfb47ece880930eb7e7635cf1f1
+    
+    # Base64 Basic Auth in test_pass_through_endpoints.py - test fixture, not a real secret
+    - name: Test Base64 Basic Auth header in pass_through_endpoints test
+      match: 61bac0491f395040617df7ef6d06029eac4d92a4457ac784978db80d97be1ae0
+    
+    # PostgreSQL password "postgres" in CI configs - standard test database password
+    - name: Test PostgreSQL password in CI configurations
+      match: 6e0d657eb1f0fbc40cf0b8f3c3873ef627cc9cb7c4108d1c07d979c04bc8a4bb
+    
+    # Bearer token in locustfile.py - test/example API key for load testing
+    - name: Test Bearer token in locustfile load test
+      match: 2a0abc2b0c3c1760a51ffcdf8d6b1d384cef69af740504b1cfa82dd70cdc7ff9
+    
+    # Inkeep API key in docusaurus.config.js - public documentation site key
+    - name: Inkeep API key in documentation config
+      match: c366657791bfb5fc69045ec11d49452f09a0aebbc8648f94e2469b4025e29a75
+    
+    # Langfuse credentials in test_completion.py - test credentials for integration test
+    - name: Langfuse test credentials in test_completion
+      match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
+    
+    # Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
+    - name: Test password in e2e test fixtures
+      match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
+    
+    # === Preventive patterns for test keys (pattern-based) ===
+    
+    # Test API keys (124 instances across 45 files)
+    - name: Test API keys with sk-test prefix
+      match: sk-test-
+    
+    # Mock API keys
+    - name: Mock API keys with sk-mock prefix
+      match: sk-mock-
+    
+    # Fake API keys
+    - name: Fake API keys with sk-fake prefix
+      match: sk-fake-
+    
+    # Generic test API key patterns
+    - name: Test API key patterns
+      match: test-api-key
+
+    - name: Short fake sk keys (1–9 digits only)
+      match: \bsk-\d{1,9}\b
+

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -7,6 +7,16 @@ body:
     attributes:
       value: |
         Thanks for taking the time to fill out this bug report!
+        
+        **💡 Tip:** See our [Troubleshooting Guide](https://docs.litellm.ai/docs/troubleshoot) for what information to include.
+  - type: checkboxes
+    id: duplicate-check
+    attributes:
+      label: Check for existing issues
+      description: Please search to see if an issue already exists for the bug you encountered.
+      options:
+        - label: I have searched the existing issues and checked that my issue is not a duplicate.
+          required: true
   - type: textarea
     id: what-happened
     attributes:
@@ -16,6 +26,21 @@ body:
       value: "A bug happened!"
     validations:
       required: true
+  - type: textarea
+    id: steps-to-reproduce
+    attributes:
+      label: Steps to Reproduce
+      description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
+      placeholder: |
+        1. config.yaml file/ .env file/ etc.
+        2. Run the following code...
+        3. Observe the error...
+      value: |
+        1. 
+        2. 
+        3. 
+    validations:
+      required: true
   - type: textarea
     id: logs
     attributes:
@@ -27,6 +52,7 @@ body:
     attributes:
       label: What part of LiteLLM is this about?
       options:
+        - ''
         - "SDK (litellm Python package)"
         - "Proxy"
         - "UI Dashboard"

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,7 @@
 blank_issues_enabled: true
 contact_links:
   - name: Schedule Demo
-    url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat
+    url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions
     about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
   - name: Discord
     url: https://discord.com/invite/wuPM9dRgDw

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -7,6 +7,14 @@ body:
     attributes:
       value: |
         Thanks for making LiteLLM better! 
+  - type: checkboxes
+    id: duplicate-check
+    attributes:
+      label: Check for existing issues
+      description: Please search to see if an issue already exists for the feature you are requesting.
+      options:
+        - label: I have searched the existing issues and checked that my issue is not a duplicate.
+          required: true
   - type: textarea
     id: the-feature
     attributes:
@@ -27,6 +35,7 @@ body:
     attributes:
       label: What part of LiteLLM is this about?
       options:
+        - ''
         - "SDK (litellm Python package)"
         - "Proxy"
         - "UI Dashboard"

.github/actions/helm-oci-chart-releaser/action.yml

@@ -40,38 +40,33 @@ outputs:
 runs:
   using: composite
   steps:
+    - name: Helm | Setup
+      uses: azure/setup-helm@v4
+      with:
+        version: v3.20.0
+
     - name: Helm | Login
       shell: bash
       run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }}
-      env:
-        HELM_EXPERIMENTAL_OCI: '1'
-    
+
     - name: Helm | Dependency
       if: inputs.update_dependencies == 'true'
       shell: bash
       run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
-      env:
-        HELM_EXPERIMENTAL_OCI: '1'
 
     - name: Helm | Package
       shell: bash
       run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }}
-      env:
-        HELM_EXPERIMENTAL_OCI: '1'
 
     - name: Helm | Push
       shell: bash
       run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }}
-      env:
-        HELM_EXPERIMENTAL_OCI: '1'
 
     - name: Helm | Logout
       shell: bash
       run: helm registry logout ${{ inputs.registry }}
-      env:
-        HELM_EXPERIMENTAL_OCI: '1'
 
     - name: Helm | Output
       id: output
       shell: bash
-      run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
+      run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT

.github/codeql/codeql-config.yml

@@ -0,0 +1,22 @@
+name: "LiteLLM CodeQL config"
+
+# Use security-extended suite instead of security-and-quality to avoid
+# result sets > 2 GiB on this codebase that cause fatal OOM failures.
+queries:
+  - uses: security-extended
+
+# These two queries are security queries included in security-extended that
+# individually produce result sets > 2 GiB on this codebase, causing fatal
+# OOM failures. Exclude them as a safety net until CI confirms they no longer
+# OOM; drop these exclusions in a follow-up once verified.
+query-filters:
+  - exclude:
+      id: py/clear-text-logging-sensitive-data  # CWE-312 — > 2 GiB result set
+  - exclude:
+      id: py/polynomial-redos                   # CWE-730 — > 2 GiB result set
+
+paths-ignore:
+  - tests
+  - docs
+  - "**/*.md"
+  - litellm/proxy/_experimental/out

.github/observatory/litellm_config.yaml

@@ -0,0 +1,19 @@
+# LiteLLM Observatory Test Configuration
+# This config is used by CI to spin up a temporary LiteLLM instance
+# for running observatory tests against RC/stable releases.
+#
+# Add model definitions for the providers you want to test.
+# Provider API keys are injected via environment variables in CI.
+
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: azure/gpt-4o
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: azure/gpt-4o-mini
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE

.github/pull_request_template.md

@@ -6,9 +6,14 @@
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
-- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
+- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
+- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
+
+## Delays in PR merge?
+
+If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slack (#pr-review)](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA).
 
 ## CI (LiteLLM team)
 

.github/scripts/close_duplicate_issues.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""
+Detect and close duplicate GitHub issues using title similarity.
+
+Modes:
+  --scan           Compare all open issues against each other (batch)
+  --issue-number N Check a single issue against older open issues
+
+Requires the `gh` CLI to be authenticated.
+"""
+
+import argparse
+import difflib
+import json
+import re
+import subprocess
+import sys
+
+
+def normalize_title(title: str) -> str:
+    """Strip common prefixes, lowercase, and collapse whitespace."""
+    title = re.sub(
+        r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
+        "",
+        title,
+        flags=re.IGNORECASE,
+    )
+    return " ".join(title.lower().split())
+
+
+def gh(*args: str) -> str:
+    """Run a gh CLI command and return stdout."""
+    result = subprocess.run(
+        ["gh", *args],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def fetch_open_issues(repo: str | None) -> list[dict]:
+    """Fetch all open issues (excluding PRs) via gh api --paginate."""
+    if repo:
+        endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    else:
+        endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    cmd = ["api", "--paginate", endpoint]
+
+    raw = gh(*cmd)
+    # gh --paginate concatenates JSON arrays, so we may get multiple arrays
+    issues = []
+    for line in raw.strip().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parsed = json.loads(line)
+        if isinstance(parsed, list):
+            issues.extend(parsed)
+        else:
+            issues.append(parsed)
+
+    # Filter out pull requests (they also appear in the issues endpoint)
+    return [i for i in issues if "pull_request" not in i]
+
+
+def close_as_duplicate(
+    issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
+) -> None:
+    """Close an issue as duplicate of another, adding a comment and label."""
+    repo_args = ["--repo", repo] if repo else []
+
+    if dry_run:
+        print(f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
+        return
+
+    # Add comment
+    comment_body = (
+        f"Closing as duplicate of #{duplicate_of}.\n\n"
+        "If you believe this is not a duplicate, please reopen and add context "
+        "explaining how this differs."
+    )
+    gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
+
+    # Add label
+    gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
+
+    # Close with not_planned reason
+    gh(
+        "api",
+        f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
+        "-X",
+        "PATCH",
+        "-f",
+        "state=closed",
+        "-f",
+        "state_reason=not_planned",
+    )
+
+    print(f"  Closed #{issue_number} as duplicate of #{duplicate_of}")
+
+
+def find_duplicate(
+    issue: dict, candidates: list[dict], threshold: float
+) -> dict | None:
+    """Return the first candidate whose normalized title is above threshold."""
+    norm = normalize_title(issue["title"])
+    for candidate in candidates:
+        if candidate["number"] == issue["number"]:
+            continue
+        cand_norm = normalize_title(candidate["title"])
+        ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
+        if ratio >= threshold:
+            return candidate
+    return None
+
+
+def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
+    """Compare every issue against all older issues. Returns count of duplicates found."""
+    # Sort oldest first
+    issues.sort(key=lambda i: i["number"])
+    closed_count = 0
+
+    for idx, issue in enumerate(issues):
+        older = issues[:idx]
+        if not older:
+            continue
+        dup = find_duplicate(issue, older, threshold)
+        if dup:
+            ratio = difflib.SequenceMatcher(
+                None,
+                normalize_title(issue["title"]),
+                normalize_title(dup["title"]),
+            ).ratio()
+            print(
+                f"#{issue['number']}: \"{issue['title']}\"\n"
+                f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+                f"({ratio:.0%} similar)"
+            )
+            close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
+            closed_count += 1
+
+    return closed_count
+
+
+def check_single(
+    issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+) -> bool:
+    """Check a single issue against all older open issues. Returns True if duplicate found."""
+    target = None
+    for i in issues:
+        if i["number"] == issue_number:
+            target = i
+            break
+
+    if target is None:
+        print(f"Issue #{issue_number} not found among open issues.")
+        return False
+
+    older = [i for i in issues if i["number"] < issue_number]
+    dup = find_duplicate(target, older, threshold)
+    if dup:
+        ratio = difflib.SequenceMatcher(
+            None,
+            normalize_title(target["title"]),
+            normalize_title(dup["title"]),
+        ).ratio()
+        print(
+            f"#{target['number']}: \"{target['title']}\"\n"
+            f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+            f"({ratio:.0%} similar)"
+        )
+        close_as_duplicate(issue_number, dup["number"], repo, dry_run)
+        return True
+
+    print(f"#{issue_number}: no duplicate found above threshold {threshold}")
+    return False
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--scan", action="store_true", help="Scan all open issues")
+    mode.add_argument("--issue-number", type=int, help="Check a single issue number")
+    parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
+    parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
+    parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
+    args = parser.parse_args()
+
+    dry_run = not args.close
+
+    if dry_run:
+        print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
+
+    print("Fetching open issues...")
+    issues = fetch_open_issues(args.repo)
+    print(f"Found {len(issues)} open issues.\n")
+
+    if args.scan:
+        count = scan_all(issues, args.threshold, args.repo, dry_run)
+        print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
+    else:
+        found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
+        sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/auto_update_price_and_context_window.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   auto_update_price_and_context_window:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/check_duplicate_issues.yml

@@ -0,0 +1,52 @@
+name: Check Duplicate Issues
+
+on:
+  issues:
+    types: [opened, edited]
+
+jobs:
+  check-duplicate:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Check for potential duplicates
+        uses: wow-actions/potential-duplicates@v1
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          label: potential-duplicate
+          threshold: 0.6
+          reaction: eyes
+          comment: |
+            **⚠️ Potential duplicate detected**
+
+            This issue appears similar to existing issue(s):
+            {{#issues}}
+            - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
+            {{/issues}}
+
+            Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
+
+      - name: Checkout close script
+        if: github.event.action == 'opened'
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        if: github.event.action == 'opened'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Auto-close if high-confidence duplicate
+        if: github.event.action == 'opened'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 .github/scripts/close_duplicate_issues.py \
+            --issue-number ${{ github.event.issue.number }} \
+            --repo ${{ github.repository }} \
+            --threshold 0.85 \
+            --close

.github/workflows/codeql.yml

@@ -0,0 +1,52 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    # Run weekly on Sundays at 04:00 UTC
+    - cron: "0 4 * * 0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: actions
+            build-mode: none
+          - language: javascript-typescript
+            build-mode: none
+          - language: python
+            build-mode: none
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          config-file: ./.github/codeql/codeql-config.yml
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/codspeed.yml

@@ -0,0 +1,44 @@
+name: CodSpeed Benchmarks
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  # Allow CodSpeed to trigger backtest performance analysis
+  # in order to generate initial data
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          pip install -e "."
+          pip install pytest pytest-codspeed==4.3.0
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v4
+        with:
+          mode: simulation
+          run: pytest tests/benchmarks/ --codspeed

.github/workflows/create_daily_staging_branch.yml

@@ -2,7 +2,7 @@ name: Create Daily Staging Branch
 
 on:
   schedule:
-    - cron: '0 0 * * *'  # Runs daily at midnight UTC
+    - cron: '0 0,12 * * *'  # Runs every 12 hours at midnight and noon UTC
   workflow_dispatch:  # Allow manual trigger
 
 jobs:
@@ -24,7 +24,43 @@ jobs:
           git config user.email "github-actions[bot]@users.noreply.github.com"
 
           # Generate branch name with MM_DD_YYYY format
-          BRANCH_NAME="litellm_staging_$(date +'%m_%d_%Y')"
+          BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
+          echo "Creating branch: $BRANCH_NAME"
+
+          # Fetch all branches
+          git fetch --all
+
+          # Check if the branch already exists
+          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+            echo "Branch $BRANCH_NAME already exists. Skipping creation."
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            # Create the new branch from main
+            git checkout -b $BRANCH_NAME origin/main
+            # Push the new branch
+            git push origin $BRANCH_NAME
+            echo "Successfully created and pushed branch: $BRANCH_NAME"
+          fi
+
+  create-internal-dev-branch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Create internal dev branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Configure Git user
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Generate branch name with MM_DD_YYYY format
+          BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
           echo "Creating branch: $BRANCH_NAME"
 
           # Fetch all branches

.github/workflows/ghcr_deploy.yml

@@ -5,6 +5,7 @@ on:
     inputs:
       tag:
         description: "The tag version you want to build"
+        required: true
       release_type:
         description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
         type: string
@@ -298,6 +299,15 @@ jobs:
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
 
+  run-observatory-tests:
+    if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable'
+    needs: [docker-hub-deploy]
+    uses: ./.github/workflows/run_observatory_tests.yml
+    with:
+      tag: ${{ github.event.inputs.tag }}
+      commit_hash: ${{ github.event.inputs.commit_hash }}
+    secrets: inherit
+
   build-and-push-helm-chart:
     if: github.event.inputs.release_type  != 'dev'
     needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
@@ -319,59 +329,37 @@ jobs:
         run: |
           echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
     
-      - name: Get LiteLLM Latest Tag
-        id: current_app_tag
-        shell: bash
-        run: |
-          LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0)
-          if [ -z "${LATEST_TAG}" ]; then
-            echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT
-          else
-            echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT
-          fi
-
-      - name: Get last published chart version
-        id: current_version
-        shell: bash
-        run: |
-          CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
-          if [ -z "${CHART_LIST}" ]; then
-            echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
-          else
-            # Extract version and strip any prerelease suffix (e.g., 0.1.827-latest -> 0.1.827)
-            VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1)
-            echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT
-          fi
-        env:
-          HELM_EXPERIMENTAL_OCI: '1'
-
-      # Automatically update the helm chart version one "patch" level
-      - name: Bump release version
-        id: bump_version
-        uses: christian-draeger/increment-semantic-version@1.1.0
-        with:
-          current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
-          version-fragment: 'bug'
-
-      # Add suffix for non-stable releases (semantic versioning)
-      - name: Calculate chart version with prerelease suffix
+      # Sync Helm chart version with LiteLLM release version (1-1 versioning)
+      # This allows users to easily map Helm chart versions to LiteLLM versions
+      # See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/
+      - name: Calculate chart and app versions
         id: chart_version
         shell: bash
         run: |
-          BASE_VERSION="${{ steps.bump_version.outputs.next-version || '0.1.0' }}"
+          INPUT_TAG="${{ github.event.inputs.tag }}"
           RELEASE_TYPE="${{ github.event.inputs.release_type }}"
-          if [ "$RELEASE_TYPE" = "stable" ]; then
-            echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT
-          else
-            echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT
+
+          # Chart version = LiteLLM version without 'v' prefix (Helm semver convention)
+          # v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1
+          CHART_VERSION="${INPUT_TAG#v}"
+
+          # Add suffix for 'latest' releases (rc already has suffix in tag)
+          if [ "$RELEASE_TYPE" = "latest" ]; then
+            CHART_VERSION="${CHART_VERSION}-latest"
           fi
 
+          # App version = Docker tag (keeps 'v' prefix to match Docker image tags)
+          APP_VERSION="${INPUT_TAG}"
+
+          echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
+          echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
+
       - uses: ./.github/actions/helm-oci-chart-releaser
         with:
           name: ${{ env.CHART_NAME }}
           repository: ${{ env.REPO_OWNER }}
-          tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '0.1.0' }}
-          app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
+          tag: ${{ steps.chart_version.outputs.version }}
+          app_version: ${{ steps.chart_version.outputs.app_version }}
           path: deploy/charts/${{ env.CHART_NAME }}
           registry: ${{ env.REGISTRY }}
           registry_username: ${{ github.actor }}
@@ -381,7 +369,8 @@ jobs:
   release:
     name: "New LiteLLM Release"
     needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
-
+    permissions:
+      contents: write
     runs-on: "ubuntu-latest"
    
     steps:

.github/workflows/ghcr_helm_deploy.yml

@@ -1,10 +1,12 @@
-# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
+# Standalone workflow to publish LiteLLM Helm Chart
+# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release
 name: Build, Publish LiteLLM Helm Chart. New Release
 on:
   workflow_dispatch:
     inputs:
-      chartVersion:
-        description: "Update the helm chart's version to this"
+      tag:
+        description: "LiteLLM version tag (e.g., v1.81.0)"
+        required: true
 
 # Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
@@ -31,24 +33,22 @@ jobs:
         run: |
           echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
 
-      - name: Get LiteLLM Latest Tag
-        id: current_app_tag
-        uses: WyriHaximus/github-action-get-previous-tag@v1.3.0
-
-      - name: Get last published chart version
-        id: current_version
+      # Sync Helm chart version with LiteLLM release version (1-1 versioning)
+      - name: Calculate chart and app versions
+        id: chart_version
         shell: bash
-        run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
-        env:
-          HELM_EXPERIMENTAL_OCI: '1'
+        run: |
+          INPUT_TAG="${{ github.event.inputs.tag }}"
 
-      # Automatically update the helm chart version one "patch" level
-      - name: Bump release version
-        id: bump_version
-        uses: christian-draeger/increment-semantic-version@1.1.0
-        with:
-          current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
-          version-fragment: 'bug'
+          # Chart version = LiteLLM version without 'v' prefix
+          # v1.81.0 -> 1.81.0
+          CHART_VERSION="${INPUT_TAG#v}"
+
+          # App version = Docker tag (keeps 'v' prefix)
+          APP_VERSION="${INPUT_TAG}"
+
+          echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
+          echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
 
       - name: Lint helm chart
         run: helm lint deploy/charts/litellm-helm
@@ -57,8 +57,8 @@ jobs:
         with:
           name: litellm-helm
           repository: ${{ env.REPO_OWNER }}
-          tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
-          app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
+          tag: ${{ steps.chart_version.outputs.version }}
+          app_version: ${{ steps.chart_version.outputs.app_version }}
           path: deploy/charts/litellm-helm
           registry: ${{ env.REGISTRY }}
           registry_username: ${{ github.actor }}

.github/workflows/interpret_load_test.py

@@ -123,7 +123,7 @@ if __name__ == "__main__":
         + docker_run_command
         + "\n\n"
         + "### Don't want to maintain your internal proxy? get in touch 🎉"
-        + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
+        + "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions"
         + "\n\n"
         + "## Load Test LiteLLM Proxy Results"
         + "\n\n"

.github/workflows/label-component.yml

@@ -11,134 +11,106 @@ jobs:
     permissions:
       issues: write
     steps:
-      - name: Add SDK label
-        if: contains(github.event.issue.body, 'SDK (litellm Python package)')
+      - name: Add component labels
         uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            const labelName = 'sdk';
-            try {
-              await github.rest.issues.getLabel({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                name: labelName
-              });
-            } catch (error) {
-              if (error.status === 404) {
-                await github.rest.issues.createLabel({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  name: labelName,
-                  color: '0E7C86',
-                  description: 'Issues related to the litellm Python SDK'
-                });
-              } else {
-                throw error;
-              }
-            }
-            await github.rest.issues.addLabels({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-              labels: [labelName]
-            });
+            const body = context.payload.issue.body;
+            if (!body) return;
 
-      - name: Add Proxy label
-        if: contains(github.event.issue.body, 'Proxy')
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const labelName = 'proxy';
-            try {
-              await github.rest.issues.getLabel({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                name: labelName
-              });
-            } catch (error) {
-              if (error.status === 404) {
-                await github.rest.issues.createLabel({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  name: labelName,
-                  color: '5319E7',
-                  description: 'Issues related to the LiteLLM Proxy'
-                });
-              } else {
-                throw error;
+            // Define component mappings with regex patterns that handle flexible whitespace
+            const components = [
+              {
+                pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/,
+                label: 'sdk',
+                color: '0E7C86',
+                description: 'Issues related to the litellm Python SDK'
+              },
+              {
+                pattern: /What part of LiteLLM is this about\?\s*Proxy/,
+                label: 'proxy',
+                color: '5319E7',
+                description: 'Issues related to the LiteLLM Proxy'
+              },
+              {
+                pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/,
+                label: 'ui-dashboard',
+                color: 'D876E3',
+                description: 'Issues related to the LiteLLM UI Dashboard'
+              },
+              {
+                pattern: /What part of LiteLLM is this about\?\s*Docs/,
+                label: 'docs',
+                color: 'FBCA04',
+                description: 'Issues related to LiteLLM documentation'
               }
-            }
-            await github.rest.issues.addLabels({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-              labels: [labelName]
-            });
+            ];
 
-      - name: Add UI Dashboard label
-        if: contains(github.event.issue.body, 'UI Dashboard')
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const labelName = 'ui-dashboard';
-            try {
-              await github.rest.issues.getLabel({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                name: labelName
-              });
-            } catch (error) {
-              if (error.status === 404) {
-                await github.rest.issues.createLabel({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  name: labelName,
-                  color: 'D876E3',
-                  description: 'Issues related to the LiteLLM UI Dashboard'
-                });
-              } else {
-                throw error;
-              }
-            }
-            await github.rest.issues.addLabels({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-              labels: [labelName]
-            });
+            // Find matching component
+            for (const component of components) {
+              if (component.pattern.test(body)) {
+                // Ensure label exists
+                try {
+                  await github.rest.issues.getLabel({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    name: component.label
+                  });
+                } catch (error) {
+                  if (error.status === 404) {
+                    await github.rest.issues.createLabel({
+                      owner: context.repo.owner,
+                      repo: context.repo.repo,
+                      name: component.label,
+                      color: component.color,
+                      description: component.description
+                    });
+                  }
+                }
 
-      - name: Add Docs label
-        if: contains(github.event.issue.body, 'Docs')
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const labelName = 'docs';
-            try {
-              await github.rest.issues.getLabel({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                name: labelName
-              });
-            } catch (error) {
-              if (error.status === 404) {
-                await github.rest.issues.createLabel({
+                // Add label to issue
+                await github.rest.issues.addLabels({
                   owner: context.repo.owner,
                   repo: context.repo.repo,
-                  name: labelName,
-                  color: 'FBCA04',
-                  description: 'Issues related to LiteLLM documentation'
+                  issue_number: context.issue.number,
+                  labels: [component.label]
                 });
-              } else {
-                throw error;
+
+                break;
               }
             }
-            await github.rest.issues.addLabels({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: context.issue.number,
-              labels: [labelName]
-            });
+
+            // Check for 'claude code' keyword (can be applied alongside component labels)
+            if (/claude code/i.test(body)) {
+              const claudeLabel = {
+                name: 'claude code',
+                color: '7c3aed',
+                description: 'Issues related to Claude Code usage'
+              };
+
+              try {
+                await github.rest.issues.getLabel({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  name: claudeLabel.name
+                });
+              } catch (error) {
+                if (error.status === 404) {
+                  await github.rest.issues.createLabel({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    name: claudeLabel.name,
+                    color: claudeLabel.color,
+                    description: claudeLabel.description
+                  });
+                }
+              }
+
+              await github.rest.issues.addLabels({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                labels: [claudeLabel.name]
+              });
+            }

.github/workflows/publish-migrations.yml

@@ -13,6 +13,7 @@ on:
 
 jobs:
   publish-migrations:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     services:
       postgres:

.github/workflows/publish_enterprise.yml

@@ -0,0 +1,94 @@
+name: Publish litellm-enterprise to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+      pull-requests: write
+    defaults:
+      run:
+        working-directory: enterprise
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump and create PR
+        id: create-pr
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}"
+          git checkout -b "$BRANCH"
+          git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push origin "$BRANCH" --force
+          gh pr create \
+            --title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \
+            --body "Version bump for litellm-enterprise. Merge to update main." \
+            --head "$BRANCH" \
+            --base main \
+          || true
+          PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url')
+          echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        run: |
+          gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}*

.github/workflows/publish_proxy_extras.yml

@@ -0,0 +1,74 @@
+name: Publish litellm-proxy-extras to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+    defaults:
+      run:
+        working-directory: litellm-proxy-extras
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}*

.github/workflows/regenerate-poetry-lock.yml

@@ -0,0 +1,80 @@
+name: Regenerate poetry.lock
+
+# Runs whenever pyproject.toml is merged into main (the most common cause of
+# the "pyproject.toml changed significantly since poetry.lock was last generated"
+# CI failure).  Can also be triggered manually.
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - pyproject.toml
+  workflow_dispatch:
+
+permissions:
+  contents: write       # needed to push the auto/regenerate-poetry-lock-* branch
+  pull-requests: write  # needed to open the PR and enable auto-merge
+
+jobs:
+  regenerate-lock:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Regenerate poetry.lock
+        run: poetry lock
+
+      - name: Check whether poetry.lock actually changed
+        id: diff
+        run: |
+          if git diff --quiet poetry.lock; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Open PR with the refreshed lock file
+        if: steps.diff.outputs.changed == 'true'
+        id: open-pr
+        run: |
+          BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')"
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$BRANCH"
+          git add poetry.lock
+          git commit -m "chore: regenerate poetry.lock to match pyproject.toml"
+          git push -f origin "$BRANCH"
+
+          cat > /tmp/pr-body.md << 'BODY'
+          Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`.
+
+          Fixes the recurring CI failure:
+          ```
+          pyproject.toml changed significantly since poetry.lock was last generated.
+          Run `poetry lock` to fix the lock file.
+          ```
+          BODY
+
+          PR_URL=$(gh pr create \
+            --title "chore: regenerate poetry.lock to match pyproject.toml" \
+            --body-file /tmp/pr-body.md \
+            --head "$BRANCH" \
+            --base main)
+          echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        if: steps.diff.outputs.changed == 'true'
+        run: |
+          gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}

.github/workflows/run_observatory_tests.yml

@@ -0,0 +1,225 @@
+name: Run Observatory Tests
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Docker image tag to test (e.g. v1.61.0.rc1)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash (defaults to HEAD of current branch)"
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      tag:
+        description: "Docker image tag to test"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash of the release"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
+
+jobs:
+  observatory-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Validate tag input
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
+            exit 1
+          fi
+
+      - name: Start LiteLLM container
+        env:
+          TAG: ${{ inputs.tag }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+        run: |
+          docker run -d \
+            --name litellm-rc \
+            -p 4000:4000 \
+            -v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
+            -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
+            -e AZURE_API_KEY="${AZURE_API_KEY}" \
+            -e AZURE_API_BASE="${AZURE_API_BASE}" \
+            "litellm/litellm:${TAG}" \
+            --config /app/config.yaml --port 4000
+
+      - name: Wait for LiteLLM health check
+        run: |
+          echo "Waiting for LiteLLM to be ready..."
+          for i in $(seq 1 30); do
+            if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
+              echo "LiteLLM is healthy"
+              exit 0
+            fi
+            echo "Attempt $i/30 - not ready yet, waiting 10s..."
+            sleep 10
+          done
+          echo "LiteLLM failed to start within 5 minutes"
+          docker logs litellm-rc
+          exit 1
+
+      - name: Start cloudflared tunnel
+        run: |
+          # Install cloudflared
+          curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
+          chmod +x /usr/local/bin/cloudflared
+
+          # Start a quick tunnel (no account needed) and capture the URL
+          cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
+          CLOUDFLARED_PID=$!
+          echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
+
+          # Wait for tunnel URL to appear in logs
+          echo "Waiting for tunnel URL..."
+          for i in $(seq 1 30); do
+            TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
+            if [ -n "$TUNNEL_URL" ]; then
+              echo "Tunnel URL: $TUNNEL_URL"
+              echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Failed to get tunnel URL"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Verify tunnel connectivity
+        run: |
+          echo "Testing tunnel at ${{ env.TUNNEL_URL }}..."
+          # Quick tunnels need time for DNS propagation; retry to avoid
+          # transient NXDOMAIN (curl exit code 6) on first attempt.
+          for i in $(seq 1 10); do
+            if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then
+              echo "Tunnel is working (attempt $i)"
+              exit 0
+            fi
+            echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
+            sleep 5
+          done
+          echo "Tunnel failed to become reachable after 50s"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Trigger observatory test run
+        id: trigger
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+        run: |
+          PAYLOAD=$(jq -n \
+            --arg url "${TUNNEL_URL}" \
+            --arg key "${LITELLM_MASTER_KEY}" \
+            '{
+              deployment_url: $url,
+              api_key: $key,
+              test_suite: "TestOAIAzureRelease",
+              models: ["gpt-4o-mini", "gpt-4o"]
+            }')
+          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
+            -H "Content-Type: application/json" \
+            -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
+            -d "$PAYLOAD")
+          HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+          BODY=$(echo "$RESPONSE" | head -n -1)
+          echo "Response ($HTTP_CODE): $BODY"
+          if [ "$HTTP_CODE" -ge 400 ]; then
+            echo "Failed to trigger test run"
+            exit 1
+          fi
+
+          # Extract request_id for polling this specific run
+          REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
+          if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
+            echo "Failed to extract request_id from response"
+            exit 1
+          fi
+          echo "Request ID: $REQUEST_ID"
+          echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
+
+      - name: Poll for test completion
+        id: poll
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+          REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
+        run: |
+          TIMEOUT=900  # 15 minutes
+          INTERVAL=30
+          ELAPSED=0
+          while [ $ELAPSED -lt $TIMEOUT ]; do
+            STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
+              -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
+            RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
+            echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
+
+            if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
+              echo "Test finished with status: $RUN_STATUS"
+              echo "$STATUS" > /tmp/observatory_result.json
+              exit 0
+            fi
+
+            sleep $INTERVAL
+            ELAPSED=$((ELAPSED + INTERVAL))
+          done
+          echo "Timed out waiting for test to complete after ${TIMEOUT}s"
+          exit 1
+
+      - name: Verify test results
+        run: |
+          RESULT=$(cat /tmp/observatory_result.json)
+          echo "Full result: $RESULT"
+
+          STATUS=$(echo "$RESULT" | jq -r '.status')
+          TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
+          FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
+          ERROR=$(echo "$RESULT" | jq -r '.error // empty')
+
+          echo "Status: $STATUS"
+          echo "Test passed: $TEST_PASSED"
+          echo "Failure rate: $FAILURE_RATE"
+
+          if [ -n "$ERROR" ]; then
+            echo "Error: $ERROR"
+          fi
+
+          if [ "$STATUS" = "failed" ]; then
+            echo "Test run failed"
+            exit 1
+          fi
+
+          if [ "$TEST_PASSED" != "true" ]; then
+            echo "Tests did not pass (failure rate: $FAILURE_RATE)"
+            exit 1
+          fi
+
+          echo "All tests passed!"
+
+      - name: Print LiteLLM logs on failure
+        if: failure()
+        run: |
+          docker logs litellm-rc 2>/dev/null || true
+          cat /tmp/cloudflared.log 2>/dev/null || true
+
+      - name: Cleanup
+        if: always()
+        run: |
+          kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true
+          docker rm -f litellm-rc 2>/dev/null || true

.github/workflows/scan_duplicate_issues.yml

@@ -0,0 +1,47 @@
+name: Scan Duplicate Issues (One-Time)
+
+on:
+  workflow_dispatch:
+    inputs:
+      threshold:
+        description: "Similarity threshold (0-1)"
+        required: false
+        default: "0.85"
+      close:
+        description: "Actually close duplicates (false = dry run)"
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout scripts
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Scan for duplicate issues
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INPUT_THRESHOLD: ${{ inputs.threshold }}
+          INPUT_CLOSE: ${{ inputs.close }}
+        run: |
+          CLOSE_FLAG=""
+          if [ "$INPUT_CLOSE" = "true" ]; then
+            CLOSE_FLAG="--close"
+          fi
+          python3 .github/scripts/close_duplicate_issues.py \
+            --scan \
+            --repo ${{ github.repository }} \
+            --threshold "$INPUT_THRESHOLD" \
+            $CLOSE_FLAG

.github/workflows/test-linting.yml

@@ -32,12 +32,11 @@ jobs:
       run: |
         poetry lock
         poetry install --with dev
-        poetry run pip install openai==1.100.1
 
-    - name: Run Black formatting
+    - name: Check Black formatting
       run: |
         cd litellm
-        poetry run black .
+        poetry run black --check --exclude '/enterprise/' .
         cd ..
 
     - name: Debug - Check file state
@@ -73,4 +72,36 @@ jobs:
 
     - name: Check import safety
       run: |
-        poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+        poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+  secret-scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Run secret scan test
+      run: |
+        pip install pytest
+        pytest tests/litellm/test_no_hardcoded_secrets.py -v
+
+    - name: Run ggshield secret scan
+      env:
+        GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
+      run: |
+        if [ -n "$GITGUARDIAN_API_KEY" ]; then
+          pip install ggshield
+          ggshield secret scan repo .
+        else
+          echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
+        fi

.github/workflows/test-litellm-matrix.yml

@@ -0,0 +1,166 @@
+name: LiteLLM Unit Tests (Matrix)
+
+on:
+  pull_request:
+    branches: [main]
+
+# Cancel in-progress runs for the same PR
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20  # Increased from 15 to 20
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group:
+          # tests/test_litellm split by subdirectory (~560 files total)
+          # Vertex AI tests separated for better isolation (prevent auth/env pollution)
+          - name: "llms-vertex"
+            path: "tests/test_litellm/llms/vertex_ai"
+            workers: 1
+            reruns: 2
+          - name: "llms-other"
+            path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+            workers: 2
+            reruns: 2
+          # tests/test_litellm/proxy split by subdirectory (~180 files total)
+          - name: "proxy-guardrails"
+            path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
+            workers: 2
+            reruns: 2
+          - name: "proxy-core"
+            path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
+            workers: 2
+            reruns: 2
+          - name: "proxy-misc"
+            path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
+            workers: 2
+            reruns: 2
+          - name: "integrations"
+            path: "tests/test_litellm/integrations"
+            workers: 2
+            reruns: 3  # Integration tests tend to be flakier
+          - name: "core-utils"
+            path: "tests/test_litellm/litellm_core_utils"
+            workers: 2
+            reruns: 1
+          - name: "other-1"
+            # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
+            path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
+            workers: 2
+            reruns: 2
+          - name: "other-2"
+            # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
+            path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
+            workers: 2
+            reruns: 2
+          - name: "other-3"
+            # remaining dirs ≈ 8.0k lines
+            path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
+            workers: 2
+            reruns: 2
+          - name: "root"
+            path: "tests/test_litellm/test_*.py"
+            workers: 2
+            reruns: 2
+          # tests/proxy_unit_tests split alphabetically (~48 files total)
+          - name: "proxy-unit-a1"
+            # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
+            path: "tests/proxy_unit_tests/test_[a-j]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-a2"
+            # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
+            path: "tests/proxy_unit_tests/test_[k-o]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b1"
+            # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
+            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b2"
+            # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
+            path: "tests/proxy_unit_tests/test_proxy_server.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b3"
+            # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
+            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b4"
+            # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
+            path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b5"
+            # proxy_token_counter (1279) - runs independently from utils
+            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b6"
+            # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
+            path: "tests/proxy_unit_tests/test_[r-t]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b7"
+            # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
+            path: "tests/proxy_unit_tests/test_[u-z]*.py"
+            workers: 2
+            reruns: 1
+
+    name: test (${{ matrix.test-group.name }})
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
+          poetry run pip install google-genai==1.22.0 \
+            google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        run: |
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run tests - ${{ matrix.test-group.name }}
+        run: |
+          poetry run pytest ${{ matrix.test-group.path }} \
+            --tb=short -vv \
+            --maxfail=10 \
+            -n ${{ matrix.test-group.workers }} \
+            --reruns ${{ matrix.test-group.reruns }} \
+            --reruns-delay 1 \
+            --dist=loadscope \
+            --durations=20

.github/workflows/test-litellm-ui-build.yml

@@ -0,0 +1,32 @@
+name: UI Build Check
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  build-ui:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: ui/litellm-dashboard
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: ui/litellm-dashboard/package-lock.json
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Build
+        run: npm run build

.github/workflows/test-litellm.yml

@@ -1,8 +1,12 @@
 name: LiteLLM Mock Tests (folder - tests/test_litellm)
 
+# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs
+# the same tests in parallel across 10 jobs for faster CI times.
+# Kept for manual debugging only.
 on:
-  pull_request:
-    branches: [ main ]
+  workflow_dispatch:  # Manual trigger only
+  # pull_request:
+  #   branches: [ main ]
 
 jobs:
   test:
@@ -34,12 +38,11 @@ jobs:
         poetry run pip install "google-genai==1.22.0"
         poetry run pip install "google-cloud-aiplatform>=1.38"
         poetry run pip install "fastapi-offline==1.7.3"
-        poetry run pip install "python-multipart==0.0.18"
+        poetry run pip install "python-multipart>=0.0.20"
+        poetry run pip install "openapi-core"
     - name: Setup litellm-enterprise as local package
       run: |
-        cd enterprise
-        poetry run pip install -e .
-        cd ..
+        poetry run pip install --force-reinstall --no-deps -e enterprise/
     - name: Run tests
       run: |
         poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

.github/workflows/test-mcp.yml

@@ -34,15 +34,13 @@ jobs:
         poetry run pip install "pytest-cov==5.0.0"
         poetry run pip install "pytest-asyncio==0.21.1"
         poetry run pip install "respx==0.22.0"
-        poetry run pip install "pydantic==2.10.2"
-        poetry run pip install "mcp==1.10.1"
+        poetry run pip install "pydantic==2.11.0"
+        poetry run pip install "mcp==1.25.0"
         poetry run pip install pytest-xdist
 
     - name: Setup litellm-enterprise as local package
       run: |
-        cd enterprise
-        python -m pip install -e .
-        cd ..
+        poetry run pip install --force-reinstall --no-deps -e enterprise/
 
     - name: Run MCP tests
       run: |

.github/workflows/test-model-map.yaml

@@ -0,0 +1,15 @@
+name: Validate model_prices_and_context_window.json
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  validate-model-prices-json:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Validate model_prices_and_context_window.json
+        run: |
+          jq empty model_prices_and_context_window.json

.github/workflows/test-proxy-e2e-azure-batches.yml

@@ -0,0 +1,90 @@
+name: Proxy E2E Azure Batches Tests
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy_e2e_azure_batches_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-e2e-batches-
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy"
+          poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        run: |
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run Prisma migrations
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+        run: |
+          cd litellm/proxy
+          poetry run prisma migrate deploy --schema schema.prisma
+          cd ../..
+
+      - name: Run Azure Batch E2E Tests
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+          USE_LOCAL_LITELLM: "true"
+          USE_MOCK_MODELS: "true"
+          USE_STATE_TRACKER: "true"
+          LITELLM_LOG: DEBUG
+        run: |
+          poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
+            -vv -s -k "test_e2e_managed_batch" \
+            --tb=short \
+            --maxfail=3 \
+            --durations=10
+

.github/workflows/test_server_root_path.yml

@@ -0,0 +1,96 @@
+name: Test Proxy SERVER_ROOT_PATH Routing
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  test-server-root-path:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    strategy:
+      matrix:
+        root_path: ["/api/v1", "/llmproxy"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/Dockerfile.non_root
+          tags: litellm-test:${{ github.sha }}
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Start LiteLLM container with SERVER_ROOT_PATH
+        run: |
+          docker run -d \
+            --name litellm-test \
+            -p 4000:4000 \
+            -e SERVER_ROOT_PATH="${{ matrix.root_path }}" \
+            -e LITELLM_MASTER_KEY="sk-1234" \
+            litellm-test:${{ github.sha }} \
+            --detailed_debug
+
+      - name: Wait for container to be healthy
+        run: |
+          echo "Waiting for LiteLLM to start..."
+          max_attempts=30
+          attempt=0
+
+          while [ $attempt -lt $max_attempts ]; do
+            if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then
+              echo "LiteLLM started successfully"
+              break
+            fi
+            attempt=$((attempt + 1))
+            echo "Attempt $attempt/$max_attempts - waiting for server to start..."
+            sleep 2
+          done
+
+          if [ $attempt -eq $max_attempts ]; then
+            echo "Server failed to start within timeout"
+            docker logs litellm-test
+            exit 1
+          fi
+
+          sleep 5
+
+      - name: Show container logs
+        if: always()
+        run: docker logs litellm-test
+
+      - name: Test UI endpoint with root path
+        run: |
+          ROOT_PATH="${{ matrix.root_path }}"
+          echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/"
+
+          for i in 1 2 3; do
+            content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/")
+            if echo "$content" | grep -q -E "(html|<!DOCTYPE|<head|<body)"; then
+              echo "UI page contains valid HTML content"
+              exit 0
+            fi
+            echo "Attempt $i/3 - no valid HTML, retrying in 5s..."
+            sleep 5
+          done
+          echo "UI page does not contain expected HTML content"
+          echo "Response: $content"
+          docker logs litellm-test
+          exit 1
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker stop litellm-test || true
+          docker rm litellm-test || true

.gitignore

@@ -1,6 +1,8 @@
 .python-version
 .venv
+.venv_policy_test
 .env
+.claude
 .newenv
 newenv/*
 litellm/proxy/myenv/*
@@ -59,9 +61,6 @@ litellm/proxy/_super_secret_config.yaml
 litellm/proxy/myenv/bin/activate
 litellm/proxy/myenv/bin/Activate.ps1
 myenv/*
-litellm/proxy/_experimental/out/404/index.html
-litellm/proxy/_experimental/out/model_hub/index.html
-litellm/proxy/_experimental/out/onboarding/index.html
 litellm/tests/log.txt
 litellm/tests/langfuse.log
 litellm/tests/langfuse.log
@@ -74,9 +73,6 @@ tests/local_testing/log.txt
 litellm/proxy/_new_new_secret_config.yaml
 litellm/proxy/custom_guardrail.py
 .mypy_cache/*
-litellm/proxy/_experimental/out/404.html
-litellm/proxy/_experimental/out/404.html
-litellm/proxy/_experimental/out/model_hub.html
 .mypy_cache/*
 litellm/proxy/application.log
 tests/llm_translation/vertex_test_account.json
@@ -93,10 +89,16 @@ tests/test_custom_dir/*
 test.py
 
 litellm_config.yaml
+!.github/observatory/litellm_config.yaml
 .cursor
 .vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*
 update_model_cost_map.py
 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
-litellm/proxy/_experimental/out/guardrails/index.html
 scripts/test_vertex_ai_search.py
+LAZY_LOADING_IMPROVEMENTS.md
+STABILIZATION_TODO.md
+**/test-results
+**/playwright-report
+**/*.storageState.json
+**/coverage

.semgrep/rules/README.md

@@ -0,0 +1,22 @@
+# Custom Semgrep rules for LiteLLM
+
+Add custom rule YAML files here. Semgrep loads all `.yml`/`.yaml` files under this directory.
+
+**Run only custom rules (CI / fail on findings):**
+
+```bash
+semgrep scan --config .semgrep/rules . --error
+```
+
+**Run with registry + custom rules:**
+
+```bash
+semgrep scan --config auto --config .semgrep/rules .
+```
+
+**Layout:**
+
+- `python/` – Python-specific rules (security, patterns)
+- Add more subdirs as needed (e.g. `generic/` for language-agnostic rules)
+
+See [Semgrep rule syntax](https://semgrep.dev/docs/writing-rules/rule-syntax/).

.semgrep/rules/python/reliability/unbounded-memory.yml

@@ -0,0 +1,17 @@
+# Unbounded memory growth – data structures without a clear max limit
+# Can lead to OOM under load.
+
+rules:
+  - id: unbounded-asyncio-queue
+    message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues).
+    severity: ERROR
+    languages: [python]
+    pattern-either:
+      - pattern: asyncio.Queue()
+      - pattern: asyncio.Queue(maxsize=0)
+    metadata:
+      category: reliability
+      cwe: "CWE-400: Uncontrolled Resource Consumption"
+      tags: [python, reliability]
+      confidence: HIGH
+      source: https://docs.python.org/3/library/asyncio-queue.html

.semgrep/rules/python/unbounded-memory.yml

@@ -0,0 +1,14 @@
+# Unbounded memory growth – data structures without a clear max limit
+# Can lead to OOM under load.
+
+rules:
+  - id: unbounded-asyncio-queue
+    message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues).
+    severity: ERROR
+    languages: [python]
+    pattern-either:
+      - pattern: asyncio.Queue()
+      - pattern: asyncio.Queue(maxsize=0)
+    metadata:
+      category: correctness
+      cwe: "CWE-400: Uncontrolled Resource Consumption"

.trivyignore

@@ -0,0 +1,12 @@
+# LiteLLM Trivy Ignore File
+# CVEs listed here are temporarily allowlisted pending fixes
+
+# Next.js vulnerabilities in UI dashboard (next@14.2.35)
+# Allowlisted: 2026-01-31, 7-day fix timeline
+# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+
+
+# HIGH: DoS via request deserialization
+GHSA-h25m-26qc-wcjf
+
+# MEDIUM: Image Optimizer DoS
+CVE-2025-59471

AGENTS.md

@@ -49,6 +49,29 @@ LiteLLM is a unified interface for 100+ LLMs that:
    - Test provider-specific functionality thoroughly
    - Consider adding load tests for performance-critical changes
 
+### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
+
+1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes**
+   - The only exception is the Tremor Table component and its required Tremor Table sub components.
+
+2. **Use Common Components as much as possible**:
+   - These are usually defined in the `common_components` directory
+   - Use these components as much as possible and avoid building new components unless needed
+
+3. **Testing**:
+   - The codebase uses **Vitest** and **React Testing Library**
+   - **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
+   - **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
+   - **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled
+   - **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present
+   - **Test names must start with "should"**: All test names should follow the pattern `it("should ...")`
+   - **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed
+   - **Structure tests properly**:
+     - First test should verify the component renders successfully
+     - Subsequent tests should focus on functionality and user interactions
+     - Use `waitFor` for async operations that aren't already awaited
+   - **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation
+
 ### IMPORTANT PATTERNS
 
 1. **Function/Tool Calling**:
@@ -86,6 +109,8 @@ Key files:
 - `litellm/proxy/auth/` - Authentication logic
 - `litellm/proxy/management_endpoints/` - Admin API endpoints
 
+**Database (proxy)**: Use Prisma model methods (`prisma_client.db.<model>.upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details.
+
 ## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
 
 LiteLLM supports MCP for agent workflows:
@@ -151,6 +176,43 @@ When opening issues or pull requests, follow these templates:
 3. **Rate Limits**: Respect provider rate limits in tests
 4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
 5. **Dependencies**: Keep dependencies minimal and well-justified
+6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
+7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
+8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift)
+
+8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
+
+   **Example of BAD** (hardcoded model checks):
+
+   ```python
+   @staticmethod
+   def _is_effort_supported_model(model: str) -> bool:
+       """Check if the model supports the output_config.effort parameter..."""
+       model_lower = model.lower()
+       if AnthropicConfig._is_claude_4_6_model(model):
+           return True
+       return any(
+           v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
+       )
+   ```
+
+   **Example of GOOD** (config-driven or helper that reads from config):
+
+   ```python
+   if (
+       "claude-3-7-sonnet" in model
+       or AnthropicConfig._is_claude_4_6_model(model)
+       or supports_reasoning(
+           model=model,
+           custom_llm_provider=self.custom_llm_provider,
+       )
+   ):
+       ...
+   ```
+
+   Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
+
+9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history.
 
 ## HELPFUL RESOURCES
 
@@ -164,4 +226,49 @@ When opening issues or pull requests, follow these templates:
 - Check similar provider implementations
 - Ensure comprehensive test coverage
 - Update documentation appropriately
-- Consider backward compatibility impact 
+- Consider backward compatibility impact
+
+## Cursor Cloud specific instructions
+
+### Environment
+
+- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
+- Python 3.12, Node 22 are pre-installed.
+- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`.
+
+### Running the proxy server
+
+Start the proxy with a config file:
+
+```bash
+poetry run litellm --config dev_config.yaml --port 4000
+```
+
+The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
+
+### Running tests
+
+See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
+
+- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
+- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
+- The `--timeout` pytest flag is NOT available; don't pass it.
+- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
+- Black `--check` may report pre-existing formatting issues; this does not block test runs.
+- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
+
+### Lint
+
+```bash
+cd litellm && poetry run ruff check .
+```
+
+Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
+
+### UI Dashboard development
+
+- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
+- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
+- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
+- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
+- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`

ARCHITECTURE.md

@@ -0,0 +1,398 @@
+# LiteLLM Architecture - LiteLLM SDK + AI Gateway
+
+This document helps contributors understand where to make changes in LiteLLM.
+
+---
+
+## How It Works
+
+The LiteLLM AI Gateway (Proxy) uses the LiteLLM SDK internally for all LLM calls:
+
+```
+OpenAI SDK (client)    ──▶  LiteLLM AI Gateway (proxy/)  ──▶  LiteLLM SDK (litellm/)  ──▶  LLM API
+Anthropic SDK (client) ──▶  LiteLLMAI Gateway (proxy/)  ──▶  LiteLLM SDK (litellm/)  ──▶  LLM API
+Any HTTP client        ──▶  LiteLLMAI Gateway (proxy/)  ──▶  LiteLLM SDK (litellm/)  ──▶  LLM API
+```
+
+The **AI Gateway** adds authentication, rate limiting, budgets, and routing on top of the SDK.
+The **SDK** handles the actual LLM provider calls, request/response transformations, and streaming.
+
+---
+
+## 1. AI Gateway (Proxy) Request Flow
+
+The AI Gateway (`litellm/proxy/`) wraps the SDK with authentication, rate limiting, and management features.
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant ProxyServer as proxy/proxy_server.py
+    participant Auth as proxy/auth/user_api_key_auth.py
+    participant Redis as Redis Cache
+    participant Hooks as proxy/hooks/
+    participant Router as router.py
+    participant Main as main.py + utils.py
+    participant Handler as llms/custom_httpx/llm_http_handler.py
+    participant Transform as llms/{provider}/chat/transformation.py
+    participant Provider as LLM Provider API
+    participant CostCalc as cost_calculator.py
+    participant LoggingObj as litellm_logging.py
+    participant DBWriter as db/db_spend_update_writer.py
+    participant Postgres as PostgreSQL
+
+    %% Request Flow
+    Client->>ProxyServer: POST /v1/chat/completions
+    ProxyServer->>Auth: user_api_key_auth()
+    Auth->>Redis: Check API key cache
+    Redis-->>Auth: Key info + spend limits
+    ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
+    Hooks->>Redis: Check/increment rate limit counters
+    ProxyServer->>Router: route_request()
+    Router->>Main: litellm.acompletion()
+    Main->>Handler: BaseLLMHTTPHandler.completion()
+    Handler->>Transform: ProviderConfig.transform_request()
+    Handler->>Provider: HTTP Request
+    Provider-->>Handler: Response
+    Handler->>Transform: ProviderConfig.transform_response()
+    Transform-->>Handler: ModelResponse
+    Handler-->>Main: ModelResponse
+    
+    %% Cost Attribution (in utils.py wrapper)
+    Main->>LoggingObj: update_response_metadata()
+    LoggingObj->>CostCalc: _response_cost_calculator()
+    CostCalc->>CostCalc: completion_cost(tokens × price)
+    CostCalc-->>LoggingObj: response_cost
+    LoggingObj-->>Main: Set response._hidden_params["response_cost"]
+    Main-->>ProxyServer: ModelResponse (with cost in _hidden_params)
+    
+    %% Response Headers + Async Logging
+    ProxyServer->>ProxyServer: Extract cost from hidden_params
+    ProxyServer->>LoggingObj: async_success_handler()
+    LoggingObj->>Hooks: async_log_success_event()
+    Hooks->>DBWriter: update_database(response_cost)
+    DBWriter->>Redis: Queue spend increment
+    DBWriter->>Postgres: Batch write spend logs (async)
+    ProxyServer-->>Client: ModelResponse + x-litellm-response-cost header
+```
+
+### Proxy Components
+
+```mermaid
+graph TD
+    subgraph "Incoming Request"
+        Client["POST /v1/chat/completions"]
+    end
+
+    subgraph "proxy/proxy_server.py"
+        Endpoint["chat_completion()"]
+    end
+
+    subgraph "proxy/auth/"
+        Auth["user_api_key_auth()"]
+    end
+
+    subgraph "proxy/"
+        PreCall["litellm_pre_call_utils.py"]
+        RouteRequest["route_llm_request.py"]
+    end
+
+    subgraph "litellm/"
+        Router["router.py"]
+        Main["main.py"]
+    end
+
+    subgraph "Infrastructure"
+        DualCache["DualCache<br/>(in-memory + Redis)"]
+        Postgres["PostgreSQL<br/>(keys, teams, spend logs)"]
+    end
+
+    Client --> Endpoint
+    Endpoint --> Auth
+    Auth --> DualCache
+    DualCache -.->|cache miss| Postgres
+    Auth --> PreCall
+    PreCall --> RouteRequest
+    RouteRequest --> Router
+    Router --> DualCache
+    Router --> Main
+    Main --> Client
+```
+
+**Key proxy files:**
+- `proxy/proxy_server.py` - Main API endpoints
+- `proxy/auth/` - Authentication (API keys, JWT, OAuth2)
+- `proxy/hooks/` - Proxy-level callbacks
+- `router.py` - Load balancing, fallbacks
+- `router_strategy/` - Routing algorithms (`lowest_latency.py`, `simple_shuffle.py`, etc.)
+
+**LLM-specific proxy endpoints:**
+
+| Endpoint | Directory | Purpose |
+|----------|-----------|---------|
+| `/v1/messages` | `proxy/anthropic_endpoints/` | Anthropic Messages API |
+| `/vertex-ai/*` | `proxy/vertex_ai_endpoints/` | Vertex AI passthrough |
+| `/gemini/*` | `proxy/google_endpoints/` | Google AI Studio passthrough |
+| `/v1/images/*` | `proxy/image_endpoints/` | Image generation |
+| `/v1/batches` | `proxy/batches_endpoints/` | Batch processing |
+| `/v1/files` | `proxy/openai_files_endpoints/` | File uploads |
+| `/v1/fine_tuning` | `proxy/fine_tuning_endpoints/` | Fine-tuning jobs |
+| `/v1/rerank` | `proxy/rerank_endpoints/` | Reranking |
+| `/v1/responses` | `proxy/response_api_endpoints/` | OpenAI Responses API |
+| `/v1/vector_stores` | `proxy/vector_store_endpoints/` | Vector stores |
+| `/*` (passthrough) | `proxy/pass_through_endpoints/` | Direct provider passthrough |
+
+**Proxy Hooks** (`proxy/hooks/__init__.py`):
+
+| Hook | File | Purpose |
+|------|------|---------|
+| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits |
+| `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
+| `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
+| `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |
+| `litellm_skills` | `proxy/hooks/skills_injection.py` | Skills injection |
+
+To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`.
+
+### Infrastructure Components
+
+The AI Gateway uses external infrastructure for persistence and caching:
+
+```mermaid
+graph LR
+    subgraph "AI Gateway (proxy/)"
+        Proxy["proxy_server.py"]
+        Auth["auth/user_api_key_auth.py"]
+        DBWriter["db/db_spend_update_writer.py<br/>DBSpendUpdateWriter"]
+        InternalCache["utils.py<br/>InternalUsageCache"]
+        CostCallback["hooks/proxy_track_cost_callback.py<br/>_ProxyDBLogger"]
+        Scheduler["APScheduler<br/>ProxyStartupEvent"]
+    end
+
+    subgraph "SDK (litellm/)"
+        Router["router.py<br/>Router.cache (DualCache)"]
+        LLMCache["caching/caching_handler.py<br/>LLMCachingHandler"]
+        CacheClass["caching/caching.py<br/>Cache"]
+    end
+
+    subgraph "Redis (caching/redis_cache.py)"
+        RateLimit["Rate Limit Counters"]
+        SpendQueue["Spend Increment Queue"]
+        KeyCache["API Key Cache"]
+        TPM_RPM["TPM/RPM Tracking"]
+        Cooldowns["Deployment Cooldowns"]
+        LLMResponseCache["LLM Response Cache"]
+    end
+
+    subgraph "PostgreSQL (proxy/schema.prisma)"
+        Keys["LiteLLM_VerificationToken"]
+        Teams["LiteLLM_TeamTable"]
+        SpendLogs["LiteLLM_SpendLogs"]
+        Users["LiteLLM_UserTable"]
+    end
+
+    Auth --> InternalCache
+    InternalCache --> KeyCache
+    InternalCache -.->|cache miss| Keys
+    InternalCache --> RateLimit
+    Router --> TPM_RPM
+    Router --> Cooldowns
+    LLMCache --> CacheClass
+    CacheClass --> LLMResponseCache
+    CostCallback --> DBWriter
+    DBWriter --> SpendQueue
+    DBWriter --> SpendLogs
+    Scheduler --> SpendLogs
+    Scheduler --> Keys
+```
+
+| Component | Purpose | Key Files/Classes |
+|-----------|---------|-------------------|
+| **Redis** | Rate limiting, API key caching, TPM/RPM tracking, cooldowns, LLM response caching, spend queuing | `caching/redis_cache.py` (`RedisCache`), `caching/dual_cache.py` (`DualCache`) |
+| **PostgreSQL** | API keys, teams, users, spend logs | `proxy/utils.py` (`PrismaClient`), `proxy/schema.prisma` |
+| **InternalUsageCache** | Proxy-level cache for rate limits + API keys (in-memory + Redis) | `proxy/utils.py` (`InternalUsageCache`) |
+| **Router.cache** | TPM/RPM tracking, deployment cooldowns, client caching (in-memory + Redis) | `router.py` (`Router.cache: DualCache`) |
+| **LLMCachingHandler** | SDK-level LLM response/embedding caching | `caching/caching_handler.py` (`LLMCachingHandler`), `caching/caching.py` (`Cache`) |
+| **DBSpendUpdateWriter** | Batches spend updates to reduce DB writes | `proxy/db/db_spend_update_writer.py` (`DBSpendUpdateWriter`) |
+| **Cost Tracking** | Calculates and logs response costs | `proxy/hooks/proxy_track_cost_callback.py` (`_ProxyDBLogger`) |
+
+**Background Jobs** (APScheduler, initialized in `proxy/proxy_server.py` → `ProxyStartupEvent.initialize_scheduled_background_jobs()`):
+
+| Job | Interval | Purpose | Key Files |
+|-----|----------|---------|-----------|
+| `update_spend` | 60s | Batch write spend logs to PostgreSQL | `proxy/db/db_spend_update_writer.py` |
+| `reset_budget` | 10-12min | Reset budgets for keys/users/teams | `proxy/management_helpers/budget_reset_job.py` |
+| `add_deployment` | 10s | Sync new model deployments from DB | `proxy/proxy_server.py` (`ProxyConfig`) |
+| `cleanup_old_spend_logs` | cron/interval | Delete old spend logs | `proxy/management_helpers/spend_log_cleanup.py` |
+| `check_batch_cost` | 30min | Calculate costs for batch jobs | `proxy/management_helpers/check_batch_cost_job.py` |
+| `check_responses_cost` | 30min | Calculate costs for responses API | `proxy/management_helpers/check_responses_cost_job.py` |
+| `process_rotations` | 1hr | Auto-rotate API keys | `proxy/management_helpers/key_rotation_manager.py` |
+| `_run_background_health_check` | continuous | Health check model deployments | `proxy/proxy_server.py` |
+| `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
+| `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
+
+**Cost Attribution Flow:**
+1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes
+2. `update_response_metadata()` (`llm_response_utils/response_metadata.py`) is called
+3. `logging_obj._response_cost_calculator()` (`litellm_logging.py`) calculates cost via `litellm.completion_cost()` (`cost_calculator.py`)
+4. Cost is stored in `response._hidden_params["response_cost"]`
+5. `proxy/common_request_processing.py` extracts cost from `hidden_params` and adds to response headers (`x-litellm-response-cost`)
+6. `logging_obj.async_success_handler()` triggers callbacks including `_ProxyDBLogger.async_log_success_event()`
+7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis
+8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s
+
+---
+
+## 2. SDK Request Flow
+
+The SDK (`litellm/`) provides the core LLM calling functionality used by both direct SDK users and the AI Gateway.
+
+```mermaid
+graph TD
+    subgraph "SDK Entry Points"
+        Completion["litellm.completion()"]
+        Messages["litellm.messages()"]
+    end
+
+    subgraph "main.py"
+        Main["completion()<br/>acompletion()"]
+    end
+
+    subgraph "utils.py"
+        GetProvider["get_llm_provider()"]
+    end
+
+    subgraph "llms/custom_httpx/"
+        Handler["llm_http_handler.py<br/>BaseLLMHTTPHandler"]
+        HTTP["http_handler.py<br/>HTTPHandler / AsyncHTTPHandler"]
+    end
+
+    subgraph "llms/{provider}/chat/"
+        TransformReq["transform_request()"]
+        TransformResp["transform_response()"]
+    end
+
+    subgraph "litellm_core_utils/"
+        Streaming["streaming_handler.py"]
+    end
+
+    subgraph "integrations/ (async, off main thread)"
+        Callbacks["custom_logger.py<br/>Langfuse, Datadog, etc."]
+    end
+
+    Completion --> Main
+    Messages --> Main
+    Main --> GetProvider
+    GetProvider --> Handler
+    Handler --> TransformReq
+    TransformReq --> HTTP
+    HTTP --> Provider["LLM Provider API"]
+    Provider --> HTTP
+    HTTP --> TransformResp
+    TransformResp --> Streaming
+    Streaming --> Response["ModelResponse"]
+    Response -.->|async| Callbacks
+```
+
+**Key SDK files:**
+- `main.py` - Entry points: `completion()`, `acompletion()`, `embedding()`
+- `utils.py` - `get_llm_provider()` resolves model → provider
+- `llms/custom_httpx/llm_http_handler.py` - Central HTTP orchestrator
+- `llms/custom_httpx/http_handler.py` - Low-level HTTP client
+- `llms/{provider}/chat/transformation.py` - Provider-specific transformations
+- `litellm_core_utils/streaming_handler.py` - Streaming response handling
+- `integrations/` - Async callbacks (Langfuse, Datadog, etc.)
+
+---
+
+## 3. Translation Layer
+
+When a request comes in, it goes through a **translation layer** that converts between API formats.
+Each translation is isolated in its own file, making it easy to test and modify independently.
+
+### Where to find translations
+
+| Incoming API | Provider | Translation File |
+|--------------|----------|------------------|
+| `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` |
+| `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` |
+| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` |
+| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
+| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
+| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
+| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
+| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
+| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
+| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
+
+### Example: Debugging prompt caching
+
+If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works:
+
+1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py`
+2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py`
+3. Compare how each handles `cache_control` in `transform_request()`
+
+### How translations work
+
+Each provider has a `Config` class that inherits from `BaseConfig` (`llms/base_llm/chat/transformation.py`):
+
+```python
+class ProviderConfig(BaseConfig):
+    def transform_request(self, model, messages, optional_params, litellm_params, headers):
+        # Convert OpenAI format → Provider format
+        return {"messages": transformed_messages, ...}
+    
+    def transform_response(self, model, raw_response, model_response, logging_obj, ...):
+        # Convert Provider format → OpenAI format
+        return ModelResponse(choices=[...], usage=Usage(...))
+```
+
+The `BaseLLMHTTPHandler` (`llms/custom_httpx/llm_http_handler.py`) calls these methods - you never need to modify the handler itself.
+
+---
+
+## 4. Adding/Modifying Providers
+
+### To add a new provider:
+
+1. Create `llms/{provider}/chat/transformation.py`
+2. Implement `Config` class with `transform_request()` and `transform_response()`
+3. Add tests in `tests/llm_translation/test_{provider}.py`
+
+### To add a feature (e.g., prompt caching):
+
+1. Find the translation file from the table above
+2. Modify `transform_request()` to handle the new parameter
+3. Add unit tests that verify the transformation
+
+### Testing checklist
+
+When adding a feature, verify it works across all paths:
+
+| Test | File Pattern |
+|------|--------------|
+| OpenAI passthrough | `tests/llm_translation/test_openai*.py` |
+| Anthropic direct | `tests/llm_translation/test_anthropic*.py` |
+| Bedrock Invoke | `tests/llm_translation/test_bedrock*.py` |
+| Bedrock Converse | `tests/llm_translation/test_bedrock*converse*.py` |
+| Vertex AI | `tests/llm_translation/test_vertex*.py` |
+| Gemini | `tests/llm_translation/test_gemini*.py` |
+
+### Unit testing translations
+
+Translations are designed to be unit testable without making API calls:
+
+```python
+from litellm.llms.bedrock.chat.converse_transformation import BedrockConverseConfig
+
+def test_prompt_caching_transform():
+    config = BedrockConverseConfig()
+    result = config.transform_request(
+        model="anthropic.claude-3-opus",
+        messages=[{"role": "user", "content": "test", "cache_control": {"type": "ephemeral"}}],
+        optional_params={},
+        litellm_params={},
+        headers={}
+    )
+    assert "cachePoint" in str(result)  # Verify cache_control was translated
+```

CLAUDE.md

@@ -90,19 +90,70 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Pydantic v2 for data validation
 - Async/await patterns throughout
 - Type hints required for all public APIs
+- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
+- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear.
+- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with.
+- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller.
+- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing.
 
 ### Testing Strategy
 - Unit tests in `tests/test_litellm/`
 - Integration tests for each provider in `tests/llm_translation/`
 - Proxy tests in `tests/proxy_unit_tests/`
 - Load tests in `tests/load_tests/`
+- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
+- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs.
+- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide.
+
+### UI / Backend Consistency
+- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
+
+### MCP OAuth / OpenAPI Transport Mapping
+- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
+- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
+- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
+- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it.
+
+### MCP Credential Storage
+- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string).
+- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair.
+- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp.
+- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints.
+
+### Browser Storage Safety (UI)
+- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS).
+- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files.
 
 ### Database Migrations
 - Prisma handles schema migrations
 - Migration files auto-generated with `prisma migrate dev`
 - Always test migrations against both PostgreSQL and SQLite
 
+### Proxy database access
+- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
+- Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
+- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory.
+- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks.
+- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing.
+- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
+- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
+- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
+- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
+
 ### Enterprise Features
 - Enterprise-specific code in `enterprise/` directory
 - Optional features enabled via environment variables
-- Separate licensing and authentication for enterprise features
+- Separate licensing and authentication for enterprise features
+
+### HTTP Client Cache Safety
+- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
+
+### Troubleshooting: DB schema out of sync after proxy restart
+`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
+
+**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
+
+**Fix options:**
+1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
+2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
+3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

CONTRIBUTING.md

@@ -7,11 +7,20 @@ Thank you for your interest in contributing to LiteLLM! We welcome contributions
 Here are the core requirements for any PR submitted to LiteLLM:
 
 - [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
+- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+
+#### Proxy (Backend) PRs
+
 - [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
 - [ ] **Ensure your PR passes all checks**:
   - [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
   - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
-- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+
+#### UI PRs
+
+- [ ] **Ensure the UI builds successfully** - `npm run build`
+- [ ] **Ensure all UI unit tests pass** - `npm run test`
+- [ ] **Add tests for new components or logic** - If you are adding a new component or new logic, add corresponding tests
 
 ## **Contributor License Agreement (CLA)**
 
@@ -245,6 +254,43 @@ docker run \
     --config /app/config.yaml --detailed_debug
 ```
 
+## UI Development
+
+### 1. Setup Your Local UI Development Environment
+
+```bash
+# Clone the repo (if you haven't already)
+git clone https://github.com/YOUR_USERNAME/litellm.git
+cd litellm
+
+# Navigate to the UI dashboard directory
+cd ui/litellm-dashboard
+
+# Install dependencies
+npm install
+
+# Start the development server
+npm run dev
+```
+
+### 2. Adding UI Tests
+
+If you are adding a **new component** or **new logic**, you must add corresponding tests.
+
+### 3. Running UI Unit Tests
+
+```bash
+npm run test
+```
+
+### 4. Building the UI
+
+Ensure the UI builds successfully before submitting your PR:
+
+```bash
+npm run build
+```
+
 ## Submitting Your PR
 
 1. **Push your branch**: `git push origin your-feature-branch`

Dockerfile

@@ -3,6 +3,7 @@ ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
 
 # Runtime image
 ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
+
 # Builder stage
 FROM $LITELLM_BUILD_IMAGE AS builder
 
@@ -20,7 +21,8 @@ RUN python -m pip install build
 COPY . .
 
 # Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+# Convert Windows line endings to Unix and make executable
+RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Build the package
 RUN rm -rf dist/* && python -m build
@@ -37,7 +39,7 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.9.0 --no-cache-dir
+RUN pip install PyJWT==2.12.0 --no-cache-dir
 
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
@@ -45,8 +47,38 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 # Ensure runtime stage runs as root
 USER root
 
-# Install runtime dependencies
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
+# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
+    # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
+    # levels inside its dependency tree. `npm install -g <pkg>` only creates a
+    # SEPARATE global package, it does NOT replace npm's internal copies.
+    # We must find and replace EVERY copy inside npm's directory.
+    GLOBAL="$(npm root -g)" && \
+    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    # SECURITY FIX: patch npm's own package.json metadata so scanners see the
+    # actual installed versions instead of the stale declared dependencies.
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    # Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
+    # no longer visible to image scanners.  The globally installed npm@latest
+    # at /usr/local/lib/node_modules/npm/ remains fully functional.
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -60,17 +92,44 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
+# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
+RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
+    if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
+
 # Remove test files and keys from dependencies
 RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
     find /usr/lib -type d -path "*/tornado/test" -delete
 
-# Install semantic_router and aurelio-sdk using script
-RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
+# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
+# Patch every copy of tar, glob, and brace-expansion inside that tree.
+RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done
 
-# Generate prisma client
-RUN prisma generate
-RUN chmod +x docker/entrypoint.sh
-RUN chmod +x docker/prod_entrypoint.sh
+# Install semantic_router and aurelio-sdk using script
+# Convert Windows line endings to Unix and make executable
+RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+
+# Generate prisma client using the correct schema
+RUN prisma generate --schema=./litellm/proxy/schema.prisma
+# Convert Windows line endings to Unix for entrypoint scripts
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
+RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
 EXPOSE 4000/tcp
 

Makefile

@@ -1,7 +1,12 @@
 # LiteLLM Makefile
 # Simple Makefile for running tests and basic development tasks
 
-.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
+.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
+	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
+	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
+	info lint lint-dev format \
+	install-dev install-proxy-dev install-test-deps \
+	install-helm-unittest check-circular-imports check-import-safety
 
 # Default target
 help:
@@ -22,9 +27,26 @@ help:
 	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
 	@echo "  make test-unit          - Run unit tests (tests/test_litellm)"
+	@echo "  make test-unit-llms     - Run LLM provider tests (~225 files)"
+	@echo "  make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
+	@echo "  make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
+	@echo "  make test-unit-proxy-misc - Run proxy misc tests (~77 files)"
+	@echo "  make test-unit-integrations - Run integration tests (~60 files)"
+	@echo "  make test-unit-core-utils - Run core utils tests (~32 files)"
+	@echo "  make test-unit-other    - Run other tests (caching, responses, etc., ~69 files)"
+	@echo "  make test-unit-root     - Run root-level tests (~34 files)"
+	@echo "  make test-proxy-unit-a  - Run proxy_unit_tests (a-o, ~20 files)"
+	@echo "  make test-proxy-unit-b  - Run proxy_unit_tests (p-z, ~28 files)"
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
 
+# Keep PIP simple for edge cases:
+PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip")
+
+# Show info
+info:
+	@echo "PIP: $(PIP)"
+
 # Installation targets
 install-dev:
 	poetry install --with dev
@@ -34,18 +56,19 @@ install-proxy-dev:
 
 # CI-compatible installations (matches GitHub workflows exactly)
 install-dev-ci:
-	pip install openai==2.8.0
+	$(PIP) install openai==2.8.0
 	poetry install --with dev
-	pip install openai==2.8.0
+	$(PIP) install openai==2.8.0
 
 install-proxy-dev-ci:
 	poetry install --with dev,proxy-dev --extras proxy
-	pip install openai==2.8.0
+	$(PIP) install openai==2.8.0
 
 install-test-deps: install-proxy-dev
-	poetry run pip install "pytest-retry==1.6.3"
-	poetry run pip install pytest-xdist
-	cd enterprise && poetry run pip install -e . && cd ..
+	poetry run $(PIP) install "pytest-retry==1.6.3"
+	poetry run $(PIP) install pytest-xdist
+	poetry run $(PIP) install openapi-core
+	cd enterprise && poetry run $(PIP) install -e . && cd ..
 
 install-helm-unittest:
 	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
@@ -61,8 +84,40 @@ format-check: install-dev
 lint-ruff: install-dev
 	cd litellm && poetry run ruff check . && cd ..
 
+# faster linter for developing ...
+# inspiration from:
+# https://github.com/astral-sh/ruff/discussions/10977
+# https://github.com/astral-sh/ruff/discussions/4049
+lint-format-changed: install-dev
+	@git diff origin/main --unified=0 --no-color -- '*.py' | \
+	perl -ne '\
+		if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
+		if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
+			$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
+			print "$$file:$$start:1-$$end:999\n"; \
+		}' | \
+	while read range; do \
+		file="$${range%%:*}"; \
+		lines="$${range#*:}"; \
+		echo "Formatting $$file (lines $$lines)"; \
+		poetry run ruff format --range "$$lines" "$$file"; \
+	done
+
+lint-ruff-dev: install-dev
+	@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
+	cd litellm && \
+	(poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \
+	poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
+	cd .. ; \
+	rm -f "$$tmpfile"
+
+lint-ruff-FULL-dev: install-dev
+	@files=$$(git diff --name-only origin/main -- '*.py'); \
+	if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \
+	else echo "No changed .py files to check."; fi
+
 lint-mypy: install-dev
-	poetry run pip install types-requests types-setuptools types-redis types-PyYAML
+	poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML
 	cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
 
 lint-black: format-check
@@ -71,11 +126,14 @@ check-circular-imports: install-dev
 	cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
 
 check-import-safety: install-dev
-	poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+	@poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
 
 # Combined linting (matches test-linting.yml workflow)
 lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
 
+# Faster linting for local development (only checks changed code)
+lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
+
 # Testing targets
 test:
 	poetry run pytest tests/
@@ -83,6 +141,38 @@ test:
 test-unit: install-test-deps
 	poetry run pytest tests/test_litellm -x -vv -n 4
 
+# Matrix test targets (matching CI workflow groups)
+test-unit-llms: install-test-deps
+	poetry run pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-guardrails: install-test-deps
+	poetry run pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-core: install-test-deps
+	poetry run pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-misc: install-test-deps
+	poetry run pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
+
+test-unit-integrations: install-test-deps
+	poetry run pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
+
+test-unit-core-utils: install-test-deps
+	poetry run pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
+
+test-unit-other: install-test-deps
+	poetry run pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
+
+test-unit-root: install-test-deps
+	poetry run pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
+
+# Proxy unit tests (tests/proxy_unit_tests split alphabetically)
+test-proxy-unit-a: install-test-deps
+	poetry run pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
+
+test-proxy-unit-b: install-test-deps
+	poetry run pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
+
 test-integration:
 	poetry run pytest tests/ -k "not test_litellm"
 
@@ -100,4 +190,4 @@ test-llm-translation-single: install-test-deps
 	@mkdir -p test-results
 	poetry run pytest tests/llm_translation/$(FILE) \
 		--junitxml=test-results/junit.xml \
-		-v --tb=short --maxfail=100 --timeout=300
+		-v --tb=short --maxfail=100 --timeout=300

README.md

@@ -2,16 +2,16 @@
         🚅 LiteLLM
     </h1>
     <p align="center">
+        <p align="center">Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.]
+        </p>
         <p align="center">
         <a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
         <a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
           <img src="https://railway.app/button.svg" alt="Deploy on Railway">
         </a>
         </p>
-        <p align="center">Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]
-        <br>
     </p>
-<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (LLM Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
+<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
 <h4 align="center">
     <a href="https://pypi.org/project/litellm/" target="_blank">
         <img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
@@ -28,29 +28,22 @@
     <a href="https://www.litellm.ai/support">
         <img src="https://img.shields.io/static/v1?label=Chat%20on&message=Slack&color=black&logo=Slack&style=flat-square" alt="Slack">
     </a>
+    <a href="https://codspeed.io/BerriAI/litellm?utm_source=badge">
+        <img src="https://img.shields.io/endpoint?url=https://codspeed.io/badge.json" alt="CodSpeed"/>
+    </a>
 </h4>
 
-LiteLLM manages:
+<img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
 
-- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
-- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
-- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
-- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
 
-LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
+## Use LiteLLM for
 
-[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
-[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
+<details open>
+<summary><b>LLMs</b> - Call 100+ LLMs (Python SDK + AI Gateway)</summary>
 
-🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
+[**All Supported Endpoints**](https://docs.litellm.ai/docs/supported_endpoints) - `/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, `/rerank`, `/a2a`, `/messages` and more.
 
-Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
-
-# Usage ([**Docs**](https://docs.litellm.ai/docs/))
-
-<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
-  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
-</a>
+### Python SDK
 
 ```shell
 pip install litellm
@@ -60,257 +53,237 @@ pip install litellm
 from litellm import completion
 import os
 
-## set ENV variables
 os.environ["OPENAI_API_KEY"] = "your-openai-key"
 os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
 
-messages = [{ "content": "Hello, how are you?","role": "user"}]
+# OpenAI
+response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hello!"}])
 
-# openai call
-response = completion(model="openai/gpt-4o", messages=messages)
-
-# anthropic call
-response = completion(model="anthropic/claude-sonnet-4-20250514", messages=messages)
-print(response)
+# Anthropic  
+response = completion(model="anthropic/claude-sonnet-4-20250514", messages=[{"role": "user", "content": "Hello!"}])
 ```
 
-### Response (OpenAI Format)
+### AI Gateway (Proxy Server)
 
-```json
-{
-    "id": "chatcmpl-1214900a-6cdd-4148-b663-b5e2f642b4de",
-    "created": 1751494488,
-    "model": "claude-sonnet-4-20250514",
-    "object": "chat.completion",
-    "system_fingerprint": null,
-    "choices": [
-        {
-            "finish_reason": "stop",
-            "index": 0,
-            "message": {
-                "content": "Hello! I'm doing well, thank you for asking. I'm here and ready to help with whatever you'd like to discuss or work on. How are you doing today?",
-                "role": "assistant",
-                "tool_calls": null,
-                "function_call": null
-            }
-        }
-    ],
-    "usage": {
-        "completion_tokens": 39,
-        "prompt_tokens": 13,
-        "total_tokens": 52,
-        "completion_tokens_details": null,
-        "prompt_tokens_details": {
-            "audio_tokens": null,
-            "cached_tokens": 0
-        },
-        "cache_creation_input_tokens": 0,
-        "cache_read_input_tokens": 0
-    }
-}
-```
-
-> **Note:** LiteLLM also supports the [Responses API](https://docs.litellm.ai/docs/response_api) (`litellm.responses()`)
-
-Call any model supported by a provider, with `model=<provider_name>/<model_name>`. There might be provider-specific details here, so refer to [provider docs for more information](https://docs.litellm.ai/docs/providers)
-
-## Async ([Docs](https://docs.litellm.ai/docs/completion/stream#async-completion))
-
-```python
-from litellm import acompletion
-import asyncio
-
-async def test_get_response():
-    user_message = "Hello, how are you?"
-    messages = [{"content": user_message, "role": "user"}]
-    response = await acompletion(model="openai/gpt-4o", messages=messages)
-    return response
-
-response = asyncio.run(test_get_response())
-print(response)
-```
-
-## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
-
-LiteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
-Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
-
-```python
-from litellm import completion
-
-messages = [{"content": "Hello, how are you?", "role": "user"}]
-
-# gpt-4o
-response = completion(model="openai/gpt-4o", messages=messages, stream=True)
-for part in response:
-    print(part.choices[0].delta.content or "")
-
-# claude sonnet 4
-response = completion('anthropic/claude-sonnet-4-20250514', messages, stream=True)
-for part in response:
-    print(part)
-```
-
-### Response chunk (OpenAI Format)
-
-```json
-{
-    "id": "chatcmpl-fe575c37-5004-4926-ae5e-bfbc31f356ca",
-    "created": 1751494808,
-    "model": "claude-sonnet-4-20250514",
-    "object": "chat.completion.chunk",
-    "system_fingerprint": null,
-    "choices": [
-        {
-            "finish_reason": null,
-            "index": 0,
-            "delta": {
-                "provider_specific_fields": null,
-                "content": "Hello",
-                "role": "assistant",
-                "function_call": null,
-                "tool_calls": null,
-                "audio": null
-            },
-            "logprobs": null
-        }
-    ],
-    "provider_specific_fields": null,
-    "stream_options": null,
-    "citations": null
-}
-```
-
-## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
-
-LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
-
-```python
-from litellm import completion
-
-## set env variables for logging tools (when using MLflow, no API key set up is required)
-os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
-os.environ["HELICONE_API_KEY"] = "your-helicone-auth-key"
-os.environ["LANGFUSE_PUBLIC_KEY"] = ""
-os.environ["LANGFUSE_SECRET_KEY"] = ""
-os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
-
-os.environ["OPENAI_API_KEY"] = "your-openai-key"
-
-# set callbacks
-litellm.success_callback = ["lunary", "mlflow", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
-
-#openai call
-response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
-```
-
-# LiteLLM Proxy Server (LLM Gateway) - ([Docs](https://docs.litellm.ai/docs/simple_proxy))
-
-Track spend + Load Balance across multiple projects
-
-[Hosted Proxy](https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy)
-
-The proxy provides:
-
-1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
-2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class)
-3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend)
-4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits)
-
-## 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/)
-
-
-## Quick Start Proxy - CLI
+[**Getting Started - E2E Tutorial**](https://docs.litellm.ai/docs/proxy/docker_quick_start) - Setup virtual keys, make your first request
 
 ```shell
 pip install 'litellm[proxy]'
+litellm --model gpt-4o
 ```
 
-### Step 1: Start litellm proxy
-
-```shell
-$ litellm --model huggingface/bigcode/starcoder
-
-#INFO: Proxy running on http://0.0.0.0:4000
-```
-
-### Step 2: Make ChatCompletions Request to Proxy
-
-
-> [!IMPORTANT]
-> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
-
 ```python
-import openai # openai v1.0.0+
-client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url
-# request sent to model set on litellm proxy, `litellm --model`
-response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
-    {
-        "role": "user",
-        "content": "this is a test request, write a short poem"
-    }
-])
+import openai
 
-print(response)
+client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000")
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Hello!"}]
+)
 ```
 
-## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys))
+[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
 
-Connect the proxy with a Postgres DB to create proxy keys
+</details>
+
+<details>
+<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
+
+[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
+
+### Python SDK - A2A Protocol
+
+```python
+from litellm.a2a_protocol import A2AClient
+from a2a.types import SendMessageRequest, MessageSendParams
+from uuid import uuid4
+
+client = A2AClient(base_url="http://localhost:10001")
+
+request = SendMessageRequest(
+    id=str(uuid4()),
+    params=MessageSendParams(
+        message={
+            "role": "user",
+            "parts": [{"kind": "text", "text": "Hello!"}],
+            "messageId": uuid4().hex,
+        }
+    )
+)
+response = await client.send_message(request)
+```
+
+### AI Gateway (Proxy Server)
+
+**Step 1.** [Add your Agent to the AI Gateway](https://docs.litellm.ai/docs/a2a#adding-your-agent)
+
+**Step 2.** Call Agent via A2A SDK
+
+```python
+from a2a.client import A2ACardResolver, A2AClient
+from a2a.types import MessageSendParams, SendMessageRequest
+from uuid import uuid4
+import httpx
+
+base_url = "http://localhost:4000/a2a/my-agent"  # LiteLLM proxy + agent name
+headers = {"Authorization": "Bearer sk-1234"}    # LiteLLM Virtual Key
+
+async with httpx.AsyncClient(headers=headers) as httpx_client:
+    resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
+    agent_card = await resolver.get_agent_card()
+    client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
+
+    request = SendMessageRequest(
+        id=str(uuid4()),
+        params=MessageSendParams(
+            message={
+                "role": "user",
+                "parts": [{"kind": "text", "text": "Hello!"}],
+                "messageId": uuid4().hex,
+            }
+        )
+    )
+    response = await client.send_message(request)
+```
+
+[**Docs: A2A Agent Gateway**](https://docs.litellm.ai/docs/a2a)
+
+</details>
+
+<details>
+<summary><b>MCP Tools</b> - Connect MCP servers to any LLM (Python SDK + AI Gateway)</summary>
+
+### Python SDK - MCP Bridge
+
+```python
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+from litellm import experimental_mcp_client
+import litellm
+
+server_params = StdioServerParameters(command="python", args=["mcp_server.py"])
+
+async with stdio_client(server_params) as (read, write):
+    async with ClientSession(read, write) as session:
+        await session.initialize()
+
+        # Load MCP tools in OpenAI format
+        tools = await experimental_mcp_client.load_mcp_tools(session=session, format="openai")
+
+        # Use with any LiteLLM model
+        response = await litellm.acompletion(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "What's 3 + 5?"}],
+            tools=tools
+        )
+```
+
+### AI Gateway - MCP Gateway
+
+**Step 1.** [Add your MCP Server to the AI Gateway](https://docs.litellm.ai/docs/mcp#adding-your-mcp)
+
+**Step 2.** Call MCP tools via `/chat/completions`
 
 ```bash
-# Get the code
-git clone https://github.com/BerriAI/litellm
-
-# Go to folder
-cd litellm
-
-# Add the master key - you can change this after setup
-echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
-
-# Add the litellm salt key - you cannot change this after adding a model
-# It is used to encrypt / decrypt your LLM API Key credentials
-# We recommend - https://1password.com/password-generator/
-# password generator to get a random hash for litellm salt key
-echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
-
-# Start
-docker compose up
+curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [{"role": "user", "content": "Summarize the latest open PR"}],
+    "tools": [{
+      "type": "mcp",
+      "server_url": "litellm_proxy/mcp/github",
+      "server_label": "github_mcp",
+      "require_approval": "never"
+    }]
+  }'
 ```
 
+### Use with Cursor IDE
 
-UI on `/ui` on your proxy server
-![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033)
-
-Set budgets and rate limits across multiple projects
-`POST /key/generate`
-
-### Request
-
-```shell
-curl 'http://0.0.0.0:4000/key/generate' \
---header 'Authorization: Bearer sk-1234' \
---header 'Content-Type: application/json' \
---data-raw '{"models": ["gpt-3.5-turbo", "gpt-4", "claude-2"], "duration": "20m","metadata": {"user": "ishaan@berri.ai", "team": "core-infra"}}'
-```
-
-### Expected Response
-
-```shell
+```json
 {
-    "key": "sk-kdEXbIqZRwEeEiHwdg7sFA", # Bearer token
-    "expires": "2023-11-19T01:38:25.838000+00:00" # datetime object
+  "mcpServers": {
+    "LiteLLM": {
+      "url": "http://localhost:4000/mcp/",
+      "headers": {
+        "x-litellm-api-key": "Bearer sk-1234"
+      }
+    }
+  }
 }
 ```
 
+[**Docs: MCP Gateway**](https://docs.litellm.ai/docs/mcp)
+
+</details>
+
+---
+
+## How to use LiteLLM
+
+You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
+
+<table style={{width: '100%', tableLayout: 'fixed'}}>
+<thead>
+<tr>
+<th style={{width: '14%'}}></th>
+<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
+<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td style={{width: '14%'}}><strong>Use Case</strong></td>
+<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
+<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
+</tr>
+<tr>
+<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
+<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
+<td style={{width: '43%'}}>Developers building LLM projects</td>
+</tr>
+<tr>
+<td style={{width: '14%'}}><strong>Key Features</strong></td>
+<td style={{width: '43%'}}>Centralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and management</td>
+<td style={{width: '43%'}}>Direct Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a>, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
+</tr>
+</tbody>
+</table>
+
+LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
+
+[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy) <br>
+[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
+
+**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
+
+Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
+
+## OSS Adopters 
+
+<table>
+  <tr>
+    <td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
+    <td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
+    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
+    <td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
+    <td><h2>Netflix</h2></td>
+    <td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
+  </tr>
+</table>
+
 ## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
 
 | Provider                                                                            | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
 |-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
+| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ |  |  |  |  |  |  |  |  |  |
 | [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |
 | [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Amazon Nova](https://docs.litellm.ai/docs/providers/amazon_nova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |  |
 | [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |  |
 | [Anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
@@ -339,7 +312,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ |  |  | ✅ |  |  |  |  |
 | [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
-| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  |  | ✅ |  |  |  |
+| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ |  |  |  |
 | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |
 | [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
@@ -417,7 +390,9 @@ curl 'http://0.0.0.0:4000/key/generate' \
 1. (In root) create virtual environment `python -m venv .venv`
 2. Activate virtual environment `source .venv/bin/activate`
 3. Install dependencies `pip install -e ".[all]"`
-4. Start proxy backend `python litellm/proxy_cli.py`
+4. `pip install prisma`
+5. `prisma generate`
+6. Start proxy backend `python litellm/proxy/proxy_cli.py`
 
 ### Frontend
 1. Navigate to `ui/litellm-dashboard`
@@ -427,7 +402,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
 # Enterprise
 For companies that need better security, user management and professional support
 
-[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 This covers:
 - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
@@ -499,4 +474,3 @@ All these checks must pass before your PR can be merged.
   <img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
 </a>
 
-

batch_small.jsonl

@@ -1,4 +0,0 @@
-{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello, how are you?"}]}}
-{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "What is the weather today?"}]}}
-{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Tell me a short joke"}]}}
-

ci_cd/.grype.yaml

@@ -0,0 +1,36 @@
+ignore:
+  - vulnerability: CVE-2026-22184
+    reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
+  # Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable
+  - vulnerability: CVE-2025-55130
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-59465
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-55131
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-59466
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2026-21637
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-55132
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: GHSA-hx9q-6w63-j58v
+    reason: orjson dumps recursion; allowlisted
+  - vulnerability: GHSA-73rr-hh4g-fpgx
+    reason: diff npm transitive dep; override in package.json, allowlisted
+  - vulnerability: CVE-2026-0865
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-15282
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2026-0672
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-15366
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-15367
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-11468
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-12781
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2026-1299
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet

ci_cd/TEST_KEY_PATTERNS.md

@@ -0,0 +1,40 @@
+# Test Key Patterns Standard
+
+Standard patterns for test/mock keys and credentials in the LiteLLM codebase to avoid triggering secret detection.
+
+## How GitGuardian Works
+
+GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
+- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
+- **High entropy** values (realistic-looking secrets) trigger detection
+- **Context-aware** detection understands code syntax like `os.environ["KEY"]`
+
+## Recommended Test Key Patterns
+
+### Option 1: Low Entropy Values (Simplest)
+These won't trigger GitGuardian's ML detector:
+
+```python
+api_key = "sk-1234"
+api_key = "sk-12345"
+database_password = "postgres"
+token = "test123"
+```
+
+### Option 2: High Entropy with Test Prefixes
+If you need realistic-looking test keys with high entropy, use these prefixes:
+
+```python
+api_key = "sk-test-abc123def456ghi789..."  # OpenAI-style test key
+api_key = "sk-mock-1234567890abcdef1234..."  # Mock key
+api_key = "sk-fake-xyz789uvw456rst123..."  # Fake key
+token = "test-api-key-with-high-entropy"
+```
+
+## Configured Ignore Patterns
+
+These patterns are in `.gitguardian.yaml` for high-entropy test keys:
+- `sk-test-*` - OpenAI-style test keys
+- `sk-mock-*` - Mock API keys  
+- `sk-fake-*` - Fake API keys
+- `test-api-key` - Generic test tokens

ci_cd/security_scans.sh

@@ -11,7 +11,7 @@ echo "Starting security scans for LiteLLM..."
 install_trivy() {
     echo "Installing Trivy and required tools..."
     sudo apt-get update
-    sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl
+    sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl bsdmainutils
     wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
     echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
     sudo apt-get update
@@ -26,15 +26,65 @@ install_grype() {
     echo "Grype installed successfully"
 }
 
+# Function to install ggshield
+install_ggshield() {
+    echo "Installing ggshield..."
+    pip3 install --upgrade pip
+    pip3 install ggshield
+    echo "ggshield installed successfully"
+}
+
+# # Function to run secret detection scans
+# run_secret_detection() {
+#     echo "Running secret detection scans..."
+    
+#     if ! command -v ggshield &> /dev/null; then
+#         install_ggshield
+#     fi
+    
+#     # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
+#     if [ -z "$GITGUARDIAN_API_KEY" ]; then
+#         echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
+#         echo "ggshield requires a GitGuardian API key to scan for secrets."
+#         echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
+#         exit 1
+#     fi
+    
+#     echo "Scanning codebase for secrets..."
+#     echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
+#     echo "ggshield will automatically handle rate limits and retry as needed."
+#     echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
+    
+#     # Use --recursive for directory scanning and auto-confirm if prompted
+#     # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
+#     # GITGUARDIAN_API_KEY environment variable will be used for authentication
+#     echo y | ggshield secret scan path . --recursive || {
+#         echo ""
+#         echo "=========================================="
+#         echo "ERROR: Secret Detection Failed"
+#         echo "=========================================="
+#         echo "ggshield has detected secrets in the codebase."
+#         echo "Please review discovered secrets above, revoke any actively used secrets"
+#         echo "from underlying systems and make changes to inject secrets dynamically at runtime."
+#         echo ""
+#         echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
+#         echo "=========================================="
+#         echo ""
+#         exit 1
+#     }
+    
+#     echo "Secret detection scans completed successfully"
+# }
+
 # Function to run Trivy scans
 run_trivy_scans() {
     echo "Running Trivy scans..."
     
     echo "Scanning LiteLLM Docs..."
-    trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
+    trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
     
     echo "Scanning LiteLLM UI..."
-    trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
+    trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
     
     echo "Trivy scans completed successfully"
 }
@@ -51,12 +101,12 @@ run_grype_scans() {
     # Build and scan Dockerfile.database
     echo "Building and scanning Dockerfile.database..."
     docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
-    grype litellm-database:latest --fail-on critical
+    grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical
     
     # Build and scan main Dockerfile
     echo "Building and scanning main Dockerfile..."
     docker build --no-cache -t litellm:latest .
-    grype litellm:latest --fail-on critical
+    grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical
     
     # Restore original .dockerignore
     echo "Restoring original .dockerignore..."
@@ -78,6 +128,41 @@ run_grype_scans() {
         "GHSA-5j98-mcp5-4vw2"
         "CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
         "CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
+        "CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
+        "CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
+        "CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
+        "CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
+        "CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
+        "CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
+        "CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
+        "CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
+        "GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
+        "GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
+        "GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
+        "GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
+        "CVE-2025-59465" # Node only used for Admin UI build/prisma
+        "CVE-2025-55131" # Node only used for Admin UI build/prisma
+        "CVE-2025-59466" # Node only used for Admin UI build/prisma
+        "CVE-2025-55130" # Node only used for Admin UI build/prisma
+        "CVE-2025-59467" # Node only used for Admin UI build/prisma
+        "CVE-2026-21637" # Node only used for Admin UI build/prisma
+        "CVE-2025-55132" # Node only used for Admin UI build/prisma
+        "GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted
+        "CVE-2025-15281" # No fix available yet
+        "CVE-2026-0865" # No fix available yet
+        "CVE-2025-15282" # No fix available yet
+        "CVE-2026-0672" # No fix available yet
+        "CVE-2025-15366" # No fix available yet
+        "CVE-2025-15367" # No fix available yet
+        "CVE-2025-12781" # No fix available yet
+        "CVE-2025-11468" # No fix available yet
+        "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
+        "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
+        "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
+        "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
+        "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
+        "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
+        "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
     )
 
     # Build JSON array of allowlisted CVE IDs for jq
@@ -158,6 +243,9 @@ main() {
     install_trivy
     install_grype
     
+    # echo "Running secret detection scans..."
+    # run_secret_detection
+    
     echo "Running filesystem vulnerability scans..."
     run_trivy_scans
     

cookbook/LiteLLM_PromptLayer.ipynb

@@ -39,7 +39,7 @@
         "import os\n",
         "os.environ['OPENAI_API_KEY'] = \"\"\n",
         "os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
-        "os.environ['PROMPTLAYER_API_KEY'] = \"pl_4ea2bb00a4dca1b8a70cebf2e9e11564\"\n",
+        "os.environ['PROMPTLAYER_API_KEY'] = \"test-promptlayer-key-123\"\n",
         "\n",
         "# Set Promptlayer as a success callback\n",
         "litellm.success_callback =['promptlayer']\n",

cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb

@@ -1,21 +1,10 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "kccfk0mHZ4Ad"
+      },
       "source": [
         "# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n",
         "\n",
@@ -32,29 +21,26 @@
         "To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
         "\n",
         "To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n"
-      ],
-      "metadata": {
-        "id": "kccfk0mHZ4Ad"
-      }
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "nmSClzCPaGH6"
+      },
       "source": [
         "## /chat/completion\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "nmSClzCPaGH6"
-      }
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### OpenAI Python SDK"
-      ],
       "metadata": {
         "id": "_vqcjwOVaKpO"
-      }
+      },
+      "source": [
+        "### OpenAI Python SDK"
+      ]
     },
     {
       "cell_type": "code",
@@ -94,15 +80,20 @@
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "## Function Calling"
-      ],
       "metadata": {
         "id": "AqkyKk9Scxgj"
-      }
+      },
+      "source": [
+        "## Function Calling"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wDg10VqLczE1"
+      },
+      "outputs": [],
       "source": [
         "from openai import OpenAI\n",
         "client = OpenAI(\n",
@@ -139,24 +130,24 @@
         ")\n",
         "\n",
         "print(completion)\n"
-      ],
-      "metadata": {
-        "id": "wDg10VqLczE1"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Azure OpenAI Python SDK"
-      ],
       "metadata": {
         "id": "YYoxLloSaNWW"
-      }
+      },
+      "source": [
+        "### Azure OpenAI Python SDK"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yA1XcgowaSRy"
+      },
+      "outputs": [],
       "source": [
         "import openai\n",
         "client = openai.AzureOpenAI(\n",
@@ -184,24 +175,24 @@
         ")\n",
         "\n",
         "print(response)"
-      ],
-      "metadata": {
-        "id": "yA1XcgowaSRy"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Langchain Python"
-      ],
       "metadata": {
         "id": "yl9qhDvnaTpL"
-      }
+      },
+      "source": [
+        "### Langchain Python"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5MUZgSquaW5t"
+      },
+      "outputs": [],
       "source": [
         "from langchain.chat_models import ChatOpenAI\n",
         "from langchain.prompts.chat import (\n",
@@ -239,24 +230,22 @@
         "response = chat(messages)\n",
         "\n",
         "print(response)"
-      ],
-      "metadata": {
-        "id": "5MUZgSquaW5t"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Curl"
-      ],
       "metadata": {
         "id": "B9eMgnULbRaz"
-      }
+      },
+      "source": [
+        "### Curl"
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "VWCCk5PFcmhS"
+      },
       "source": [
         "\n",
         "\n",
@@ -280,22 +269,24 @@
         "}'\n",
         "```\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "VWCCk5PFcmhS"
-      }
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### LlamaIndex"
-      ],
       "metadata": {
         "id": "drBAm2e1b6xe"
-      }
+      },
+      "source": [
+        "### LlamaIndex"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d0bZcv8fb9mL"
+      },
+      "outputs": [],
       "source": [
         "import os, dotenv\n",
         "\n",
@@ -326,24 +317,24 @@
         "query_engine = index.as_query_engine()\n",
         "response = query_engine.query(\"What did the author do growing up?\")\n",
         "print(response)\n"
-      ],
-      "metadata": {
-        "id": "d0bZcv8fb9mL"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Langchain JS"
-      ],
       "metadata": {
         "id": "xypvNdHnb-Yy"
-      }
+      },
+      "source": [
+        "### Langchain JS"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R55mK2vCcBN2"
+      },
+      "outputs": [],
       "source": [
         "import { ChatOpenAI } from \"@langchain/openai\";\n",
         "\n",
@@ -359,24 +350,24 @@
         "const message = await model.invoke(\"Hi there!\");\n",
         "\n",
         "console.log(message);\n"
-      ],
-      "metadata": {
-        "id": "R55mK2vCcBN2"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### OpenAI JS"
-      ],
       "metadata": {
         "id": "nC4bLifCcCiW"
-      }
+      },
+      "source": [
+        "### OpenAI JS"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MICH8kIMcFpg"
+      },
+      "outputs": [],
       "source": [
         "const { OpenAI } = require('openai');\n",
         "\n",
@@ -398,24 +389,24 @@
         "}\n",
         "\n",
         "main();\n"
-      ],
-      "metadata": {
-        "id": "MICH8kIMcFpg"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Anthropic SDK"
-      ],
       "metadata": {
         "id": "D1Q07pEAcGTb"
-      }
+      },
+      "source": [
+        "### Anthropic SDK"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qBjFcAvgcI3t"
+      },
+      "outputs": [],
       "source": [
         "import os\n",
         "\n",
@@ -423,7 +414,7 @@
         "\n",
         "client = Anthropic(\n",
         "    base_url=\"http://localhost:4000\", # proxy endpoint\n",
-        "    api_key=\"sk-s4xN1IiLTCytwtZFJaYQrA\", # litellm proxy virtual key\n",
+        "    api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n",
         ")\n",
         "\n",
         "message = client.messages.create(\n",
@@ -437,33 +428,33 @@
         "    model=\"claude-3-opus-20240229\",\n",
         ")\n",
         "print(message.content)"
-      ],
-      "metadata": {
-        "id": "qBjFcAvgcI3t"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "## /embeddings"
-      ],
       "metadata": {
         "id": "dFAR4AJGcONI"
-      }
+      },
+      "source": [
+        "## /embeddings"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### OpenAI Python SDK"
-      ],
       "metadata": {
         "id": "lgNoM281cRzR"
-      }
+      },
+      "source": [
+        "### OpenAI Python SDK"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NY3DJhPfcQhA"
+      },
+      "outputs": [],
       "source": [
         "import openai\n",
         "from openai import OpenAI\n",
@@ -478,24 +469,24 @@
         ")\n",
         "\n",
         "print(response)\n"
-      ],
-      "metadata": {
-        "id": "NY3DJhPfcQhA"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Langchain Embeddings"
-      ],
       "metadata": {
         "id": "hmbg-DW6cUZs"
-      }
+      },
+      "source": [
+        "### Langchain Embeddings"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lX2S8Nl1cWVP"
+      },
+      "outputs": [],
       "source": [
         "from langchain.embeddings import OpenAIEmbeddings\n",
         "\n",
@@ -526,24 +517,22 @@
         "\n",
         "print(f\"TITAN EMBEDDINGS\")\n",
         "print(query_result[:5])"
-      ],
-      "metadata": {
-        "id": "lX2S8Nl1cWVP"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Curl Request"
-      ],
       "metadata": {
         "id": "oqGbWBCQcYfd"
-      }
+      },
+      "source": [
+        "### Curl Request"
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "7rkIMV9LcdwQ"
+      },
       "source": [
         "\n",
         "\n",
@@ -556,10 +545,21 @@
         "  }'\n",
         "```\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "7rkIMV9LcdwQ"
-      }
+      ]
     }
-  ]
-}
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md

@@ -0,0 +1,295 @@
+# Claude Code with LiteLLM Quickstart
+
+This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code.
+
+> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
+
+## Video Walkthrough
+
+Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56
+
+## Prerequisites
+
+- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
+- API keys for your chosen providers
+
+## Installation
+
+First, install LiteLLM with proxy support:
+
+```bash
+pip install 'litellm[proxy]'
+```
+
+## Step 1: Setup config.yaml
+
+Create a secure configuration using environment variables:
+
+```yaml
+model_list:
+  # Claude models
+  - model_name: claude-3-5-sonnet-20241022    
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+  
+  - model_name: claude-3-5-haiku-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-haiku-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+Set your environment variables:
+
+```bash
+export ANTHROPIC_API_KEY="your-anthropic-api-key"
+export LITELLM_MASTER_KEY="sk-1234567890"  # Generate a secure key
+```
+
+## Step 2: Start Proxy
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+## Step 3: Verify Setup
+
+Test that your proxy is working correctly:
+
+```bash
+curl -X POST http://0.0.0.0:4000/v1/messages \
+-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+-H "Content-Type: application/json" \
+-d '{
+    "model": "claude-3-5-sonnet-20241022",
+    "max_tokens": 1000,
+    "messages": [{"role": "user", "content": "What is the capital of France?"}]
+}'
+```
+
+## Step 4: Configure Claude Code
+
+### Method 1: Unified Endpoint (Recommended)
+
+Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here:
+
+```bash
+export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
+export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
+```
+
+> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI.
+
+### Method 2: Provider-specific Pass-through Endpoint
+
+Alternatively, use the Anthropic pass-through endpoint:
+
+```bash
+export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
+export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
+```
+
+## Step 5: Use Claude Code
+
+### Choosing Your Model
+
+You have two options for specifying which model Claude Code uses:
+
+#### Option 1: Command Line / Session Model Selection
+
+Specify the model directly when starting Claude Code or during a session:
+
+```bash
+# Specify model at startup
+claude --model claude-3-5-sonnet-20241022
+
+# Or change model during a session
+/model claude-3-5-haiku-20241022
+```
+
+This method uses the exact model you specify.
+
+#### Option 2: Environment Variables
+
+Configure default models using environment variables:
+
+```bash
+# Tell Claude Code which models to use by default
+export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
+export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
+export ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-3-5-20240229
+
+claude  # Will use the models specified above
+```
+
+**Note:** Claude Code may cache the model from a previous session. If environment variables don't take effect, use Option 1 to explicitly set the model.
+
+**Important:** The `model_name` in your LiteLLM config must match what Claude Code requests (either from env vars or command line).
+
+### Using 1M Context Window
+
+Claude Code supports extended context (1 million tokens) using the `[1m]` suffix with Claude 4+ models:
+
+```bash
+# Use Sonnet 4.5 with 1M context (requires quotes for shell)
+claude --model 'claude-sonnet-4-5-20250929[1m]'
+
+# Inside a Claude Code session (no quotes needed)
+/model claude-sonnet-4-5-20250929[1m]
+```
+
+**Important:** When using `--model` with `[1m]` in the shell, you must use quotes to prevent the shell from interpreting the brackets.
+
+Alternatively, set as default with environment variables:
+
+```bash
+export ANTHROPIC_DEFAULT_SONNET_MODEL='claude-sonnet-4-5-20250929[1m]'
+claude
+```
+
+**How it works:**
+- Claude Code strips the `[1m]` suffix before sending to LiteLLM
+- Claude Code automatically adds the header `anthropic-beta: context-1m-2025-08-07`
+- Your LiteLLM config should **NOT** include `[1m]` in model names
+
+**Verify 1M context is active:**
+```bash
+/context
+# Should show: 21k/1000k tokens (2%)
+```
+
+**Pricing:** Models using 1M context have different pricing. Input tokens above 200k are charged at a higher rate.
+
+## Troubleshooting
+
+Common issues and solutions:
+
+**Claude Code not connecting:**
+- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
+- Check that `ANTHROPIC_BASE_URL` is set correctly
+- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
+
+**Authentication errors:**
+- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
+- Check that your API keys are valid and have sufficient credits
+- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
+
+**Model not found:**
+- Check what model Claude Code is requesting in LiteLLM logs
+- Ensure your `config.yaml` has a matching `model_name` entry
+- If using environment variables, verify they're set: `echo $ANTHROPIC_DEFAULT_SONNET_MODEL`
+
+**1M context not working (showing 200k instead of 1000k):**
+- Verify you're using the `[1m]` suffix: `/model your-model-name[1m]`
+- Check LiteLLM logs for the header `context-1m-2025-08-07` in the request
+- Ensure your model supports 1M context (only certain Claude models do)
+- Your LiteLLM config should **NOT** include `[1m]` in the `model_name`
+
+## Using Multiple Models and Providers
+
+You can configure LiteLLM to route to any supported provider. Here's an example with multiple providers:
+
+```yaml
+model_list:
+  # OpenAI models
+  - model_name: codex-mini
+    litellm_params:
+      model: openai/codex-mini
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: https://api.openai.com/v1
+
+  - model_name: o3-pro
+    litellm_params:
+      model: openai/o3-pro
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: https://api.openai.com/v1
+
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: https://api.openai.com/v1
+
+  # Anthropic models
+  - model_name: claude-3-5-sonnet-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: claude-3-5-haiku-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-haiku-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  # AWS Bedrock
+  - model_name: claude-bedrock
+    litellm_params:
+      model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-east-1
+
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+**Note:** The `model_name` can be anything you choose. Claude Code will request whatever model you specify (via env vars or command line), and LiteLLM will route to the `model` configured in `litellm_params`.
+
+Switch between models seamlessly:
+
+```bash
+# Use environment variables to set defaults
+export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
+export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
+
+# Or specify directly
+claude --model claude-3-5-sonnet-20241022  # Complex reasoning
+claude --model claude-3-5-haiku-20241022    # Fast responses
+claude --model claude-bedrock                # Bedrock deployment
+```
+
+## Default Models Used by Claude Code
+
+If you **don't** set environment variables, Claude Code uses these default model names:
+
+| Purpose | Default Model Name (v2.1.14) |
+|---------|------------------------------|
+| Main model | `claude-sonnet-4-5-20250929` |
+| Light tasks (subagents, summaries) | `claude-haiku-4-5-20251001` |
+| Planning mode | `claude-opus-4-5-20251101` |
+
+Your LiteLLM config should include these model names if you want Claude Code to work without setting environment variables:
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-5-20250929
+    litellm_params:
+      # Can be any provider - Anthropic, Bedrock, Vertex AI, etc.
+      model: anthropic/claude-sonnet-4-5-20250929
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: claude-haiku-4-5-20251001
+    litellm_params:
+      model: anthropic/claude-haiku-4-5-20251001
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: claude-opus-4-5-20251101
+    litellm_params:
+      model: anthropic/claude-opus-4-5-20251101
+      api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+**Warning:** These default model names may change with new Claude Code versions. Check LiteLLM proxy logs for "model not found" errors to identify what Claude Code is requesting.
+
+## Additional Resources
+
+- [LiteLLM Documentation](https://docs.litellm.ai/)
+- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
+- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)
+

cookbook/ai_coding_tool_guides/index.json

@@ -0,0 +1,134 @@
+[{
+    "title": "Claude Code Quickstart",
+    "description": "This is a quickstart guide to using Claude Code with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api",
+    "date": "2026-01-15",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM"
+    ]
+},
+{
+    "title": "Claude Code with MCPs",
+    "description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_mcp",
+    "date": "2026-01-15",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM",
+        "MCP"
+    ]
+}, 
+{
+    "title": "Claude Code with Non-Anthropic Models",
+    "description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM",
+        "OpenAI",
+        "Gemini"
+    ]
+},
+{
+    "title": "Cursor Quickstart",
+    "description": "This is a quickstart guide to using Cursor with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/cursor_integration",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Cursor",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "Github Copilot Quickstart",
+    "description": "This is a quickstart guide to using Github Copilot with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Github Copilot",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "LiteLLM Gemini CLI Quickstart",
+    "description": "This is a quickstart guide to using LiteLLM Gemini CLI.",
+    "url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Gemini CLI",
+        "Gemini",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "OpenAI Codex CLI Quickstart",
+    "description": "This is a quickstart guide to using OpenAI Codex CLI.",
+    "url": "https://docs.litellm.ai/docs/tutorials/openai_codex",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "OpenAI Codex CLI",
+        "OpenAI",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "OpenWebUI Quickstart",
+    "description": "This is a quickstart guide to using OpenWebUI with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/openweb_ui",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "OpenWebUI",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "AI Coding Tool Usage Tracking",
+    "description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding",
+    "date": "2026-01-17",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "Gemini CLI",
+        "OpenAI Codex",
+        "LiteLLM"
+    ]
+},
+{
+    "title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)",
+    "description": "This is a guide for using Web Search with Claude Code via LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch",
+    "date": "2026-01-17",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM",
+        "Web Search"
+    ]
+},
+{
+    "title": "Track Claude Code Usage per user via Custom Headers",
+    "description": "This is a guide for tracking claude code user usage by passing a customer ID header.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking",
+    "date": "2026-01-17",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM"
+    ]
+}]

cookbook/anthropic_agent_sdk/README.md

@@ -0,0 +1,144 @@
+# Claude Agent SDK with LiteLLM Gateway
+
+A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK.
+
+## Quick Start
+
+### 1. Install dependencies
+
+```bash
+pip install anthropic claude-agent-sdk litellm
+```
+
+### 2. Start LiteLLM proxy
+
+```bash
+# Simple start with Claude
+litellm --model claude-sonnet-4-20250514
+
+# Or with a config file
+litellm --config config.yaml
+```
+
+### 3. Run the chat
+
+**Basic Agent (no MCP):**
+
+```bash
+python main.py
+```
+
+**Agent with MCP (DeepWiki2 for research):**
+
+```bash
+python agent_with_mcp.py
+```
+
+If MCP connection fails, you can disable it:
+
+```bash
+USE_MCP=false python agent_with_mcp.py
+```
+
+That's it! You can now chat with the agent in your terminal.
+
+### Chat Commands
+
+While chatting, you can use these commands:
+- `models` - List all available models (fetched from your LiteLLM proxy)
+- `model` - Switch to a different model
+- `clear` - Start a new conversation
+- `quit` or `exit` - End the chat
+
+The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured.
+
+## Configuration
+
+Set these environment variables if needed:
+
+```bash
+export LITELLM_PROXY_URL="http://localhost:4000"
+export LITELLM_API_KEY="sk-1234"
+export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
+```
+
+Or just use the defaults - it'll connect to `http://localhost:4000` by default.
+
+## Files
+
+- `main.py` - Basic interactive agent without MCP
+- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2)
+- `common.py` - Shared utilities and functions
+- `config.example.yaml` - Example LiteLLM configuration
+- `requirements.txt` - Python dependencies
+
+## Example Config File
+
+If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`):
+
+```yaml
+model_list:
+  - model_name: bedrock-claude-sonnet-4
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+      aws_region_name: "us-east-1"
+```
+
+Then start LiteLLM with: `litellm --config config.yaml`
+
+## How It Works
+
+The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
+
+```python
+# Point to LiteLLM gateway (not Anthropic)
+os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
+os.environ["ANTHROPIC_API_KEY"] = "sk-1234"  # Your LiteLLM key
+
+# Use any model configured in LiteLLM
+options = ClaudeAgentOptions(
+    model="bedrock-claude-sonnet-4",  # or gpt-4, or anything else
+    system_prompt="You are a helpful assistant.",
+    max_turns=50,
+)
+```
+
+Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically.
+
+## Why Use This?
+
+- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc.
+- **Cost tracking**: LiteLLM tracks spending across all your agent conversations
+- **Rate limiting**: Set budgets and limits on your agent usage
+- **Load balancing**: Distribute requests across multiple API keys or regions
+- **Fallbacks**: Automatically retry with a different model if one fails
+
+## Troubleshooting
+
+**Connection errors?**
+- Make sure LiteLLM is running: `litellm --model your-model`
+- Check the URL is correct (default: `http://localhost:4000`)
+
+**Authentication errors?**
+- Verify your LiteLLM API key is correct
+- Make sure the model is configured in your LiteLLM setup
+
+**Model not found?**
+- Check the model name matches what's in your LiteLLM config
+- Run `litellm --model your-model` to test it works
+
+**Agent with MCP stuck or failing?**
+- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2`
+- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py`
+- Or use the basic agent: `python main.py`
+
+## Learn More
+
+- [LiteLLM Docs](https://docs.litellm.ai/)
+- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk)
+- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start)

cookbook/anthropic_agent_sdk/agent_with_mcp.py

@@ -0,0 +1,140 @@
+"""
+Interactive Claude Agent SDK CLI with MCP Support
+
+This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy,
+with MCP (Model Context Protocol) server integration for enhanced capabilities.
+"""
+
+import asyncio
+import os
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+from common import (
+    Config,
+    fetch_available_models,
+    setup_litellm_env,
+    print_header,
+    handle_model_list,
+    handle_model_switch,
+    stream_response,
+)
+
+
+async def interactive_chat_with_mcp():
+    """
+    Interactive CLI chat with the agent and MCP server
+    """
+    config = Config()
+    
+    # Configure Anthropic SDK to point to LiteLLM gateway
+    litellm_base_url = setup_litellm_env(config)
+    
+    # Fetch available models from proxy
+    available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
+    
+    current_model = config.LITELLM_MODEL
+    
+    # MCP server configuration
+    mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
+    use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
+    
+    if not use_mcp:
+        print("⚠️  MCP disabled via USE_MCP=false")
+    
+    print_header(litellm_base_url, current_model, has_mcp=use_mcp)
+    
+    while True:
+        # Configure agent options
+        if use_mcp:
+            try:
+                # Try with MCP server (HTTP transport)
+                # Using McpHttpServerConfig format from Agent SDK
+                options = ClaudeAgentOptions(
+                    system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.",
+                    model=current_model,
+                    max_turns=50,
+                    mcp_servers={
+                        "deepwiki2": {
+                            "type": "http",
+                            "url": mcp_server_url,
+                            "headers": {
+                                "Authorization": f"Bearer {config.LITELLM_API_KEY}"
+                            }
+                        }
+                    },
+                )
+            except Exception as e:
+                print(f"⚠️  Warning: Could not configure MCP server: {e}")
+                print("Continuing without MCP...\n")
+                use_mcp = False
+                options = ClaudeAgentOptions(
+                    system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+                    model=current_model,
+                    max_turns=50,
+                )
+        else:
+            # Without MCP
+            options = ClaudeAgentOptions(
+                system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+                model=current_model,
+                max_turns=50,
+            )
+        
+        # Create agent client
+        try:
+            async with ClaudeSDKClient(options=options) as client:
+                conversation_active = True
+                
+                while conversation_active:
+                    # Get user input
+                    try:
+                        user_input = input("\n👤 You: ").strip()
+                    except (EOFError, KeyboardInterrupt):
+                        print("\n\n👋 Goodbye!")
+                        return
+                    
+                    # Handle commands
+                    if user_input.lower() in ['quit', 'exit']:
+                        print("\n👋 Goodbye!")
+                        return
+                    
+                    if user_input.lower() == 'clear':
+                        print("\n🔄 Starting new conversation...\n")
+                        conversation_active = False
+                        continue
+                    
+                    if user_input.lower() == 'models':
+                        handle_model_list(available_models, current_model)
+                        continue
+                    
+                    if user_input.lower() == 'model':
+                        new_model, should_restart = handle_model_switch(available_models, current_model)
+                        if should_restart:
+                            current_model = new_model
+                            conversation_active = False
+                        continue
+                    
+                    if not user_input:
+                        continue
+                    
+                    # Stream response from agent
+                    await stream_response(client, user_input)
+        
+        except Exception as e:
+            print(f"\n❌ Error creating agent client: {e}")
+            print("This might be an MCP configuration issue. Try running without MCP:")
+            print("  USE_MCP=false python agent_with_mcp.py")
+            print("\nOr use the basic agent:")
+            print("  python main.py")
+            return
+
+
+def main():
+    """Run interactive chat with MCP"""
+    try:
+        asyncio.run(interactive_chat_with_mcp())
+    except KeyboardInterrupt:
+        print("\n\n👋 Goodbye!")
+
+
+if __name__ == "__main__":
+    main()

cookbook/anthropic_agent_sdk/common.py

@@ -0,0 +1,160 @@
+"""
+Common utilities for Claude Agent SDK examples
+"""
+
+import os
+import httpx
+
+
+class Config:
+    """Configuration for LiteLLM Gateway connection"""
+    
+    # LiteLLM proxy URL (default to local instance)
+    LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
+    
+    # LiteLLM API key (master key or virtual key)
+    LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
+    
+    # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
+    LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
+
+
+async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
+    """
+    Fetch available models from LiteLLM proxy /models endpoint
+    """
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(
+                f"{base_url}/models",
+                headers={"Authorization": f"Bearer {api_key}"},
+                timeout=10.0
+            )
+            response.raise_for_status()
+            data = response.json()
+            return [model["id"] for model in data.get("data", [])]
+    except Exception as e:
+        print(f"⚠️  Warning: Could not fetch models from proxy: {e}")
+        print("Using default model list...")
+        # Fallback to default models
+        return [
+            "bedrock-claude-sonnet-3.5",
+            "bedrock-claude-sonnet-4",
+            "bedrock-claude-sonnet-4.5",
+            "bedrock-claude-opus-4.5",
+            "bedrock-nova-premier",
+        ]
+
+
+def setup_litellm_env(config: Config):
+    """
+    Configure environment variables to point Agent SDK to LiteLLM
+    """
+    litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
+    os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
+    os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
+    return litellm_base_url
+
+
+def print_header(base_url: str, current_model: str, has_mcp: bool = False):
+    """
+    Print the chat header
+    """
+    mcp_indicator = " + MCP" if has_mcp else ""
+    print("=" * 70)
+    print(f"🤖 Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat")
+    print("=" * 70)
+    print(f"🚀 Connected to: {base_url}")
+    print(f"📦 Current model: {current_model}")
+    if has_mcp:
+        print("🔌 MCP: deepwiki2 enabled")
+    print("\nType your messages below. Commands:")
+    print("  - 'quit' or 'exit' to end the conversation")
+    print("  - 'clear' to start a new conversation")
+    print("  - 'model' to switch models")
+    print("  - 'models' to list available models")
+    print("=" * 70)
+    print()
+
+
+def handle_model_list(available_models: list[str], current_model: str):
+    """
+    Display available models
+    """
+    print("\n📋 Available models:")
+    for i, model in enumerate(available_models, 1):
+        marker = "✓" if model == current_model else " "
+        print(f"  {marker} {i}. {model}")
+
+
+def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
+    """
+    Handle model switching
+    
+    Returns:
+        tuple: (new_model, should_restart_conversation)
+    """
+    print("\n📋 Select a model:")
+    for i, model in enumerate(available_models, 1):
+        marker = "✓" if model == current_model else " "
+        print(f"  {marker} {i}. {model}")
+    
+    try:
+        choice = input("\nEnter number (or press Enter to cancel): ").strip()
+        if choice:
+            idx = int(choice) - 1
+            if 0 <= idx < len(available_models):
+                new_model = available_models[idx]
+                print(f"\n✅ Switched to: {new_model}")
+                print("🔄 Starting new conversation with new model...\n")
+                return new_model, True
+            else:
+                print("❌ Invalid choice")
+    except (ValueError, IndexError):
+        print("❌ Invalid input")
+    
+    return current_model, False
+
+
+async def stream_response(client, user_input: str):
+    """
+    Stream response from the agent
+    """
+    print("\n🤖 Assistant: ", end='', flush=True)
+    
+    try:
+        await client.query(user_input)
+        
+        # Show loading indicator
+        print("⏳ thinking...", end='', flush=True)
+        
+        # Stream the response
+        first_chunk = True
+        async for msg in client.receive_response():
+            # Clear loading indicator on first message
+            if first_chunk:
+                print("\r🤖 Assistant: ", end='', flush=True)
+                first_chunk = False
+            
+            # Handle different message types
+            if hasattr(msg, 'type'):
+                if msg.type == 'content_block_delta':
+                    # Streaming text delta
+                    if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
+                        print(msg.delta.text, end='', flush=True)
+                elif msg.type == 'content_block_start':
+                    # Start of content block
+                    if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
+                        print(msg.content_block.text, end='', flush=True)
+            
+            # Fallback to original content handling
+            if hasattr(msg, 'content'):
+                for content_block in msg.content:
+                    if hasattr(content_block, 'text'):
+                        print(content_block.text, end='', flush=True)
+        
+        print()  # New line after response
+        
+    except Exception as e:
+        print(f"\r\n❌ Error: {e}")
+        print("Please check your LiteLLM gateway is running and configured correctly.")

cookbook/anthropic_agent_sdk/config.example.yaml

@@ -0,0 +1,25 @@
+model_list:
+  - model_name: bedrock-claude-sonnet-3.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-opus-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-nova-premier
+    litellm_params:
+      model: "bedrock/amazon.nova-premier-v1:0"
+      aws_region_name: "us-east-1"

cookbook/anthropic_agent_sdk/main.py

@@ -0,0 +1,95 @@
+"""
+Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway
+
+This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy.
+LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.)
+through the Claude Agent SDK by pointing it to the LiteLLM gateway.
+"""
+
+import asyncio
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+from common import (
+    Config,
+    fetch_available_models,
+    setup_litellm_env,
+    print_header,
+    handle_model_list,
+    handle_model_switch,
+    stream_response,
+)
+
+
+async def interactive_chat():
+    """
+    Interactive CLI chat with the agent
+    """
+    config = Config()
+    
+    # Configure Anthropic SDK to point to LiteLLM gateway
+    litellm_base_url = setup_litellm_env(config)
+    
+    # Fetch available models from proxy
+    available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
+    
+    current_model = config.LITELLM_MODEL
+    
+    print_header(litellm_base_url, current_model)
+    
+    while True:
+        # Configure agent options for each conversation
+        options = ClaudeAgentOptions(
+            system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+            model=current_model,
+            max_turns=50,
+        )
+        
+        # Create agent client
+        async with ClaudeSDKClient(options=options) as client:
+            conversation_active = True
+            
+            while conversation_active:
+                # Get user input
+                try:
+                    user_input = input("\n👤 You: ").strip()
+                except (EOFError, KeyboardInterrupt):
+                    print("\n\n👋 Goodbye!")
+                    return
+                
+                # Handle commands
+                if user_input.lower() in ['quit', 'exit']:
+                    print("\n👋 Goodbye!")
+                    return
+                
+                if user_input.lower() == 'clear':
+                    print("\n🔄 Starting new conversation...\n")
+                    conversation_active = False
+                    continue
+                
+                if user_input.lower() == 'models':
+                    handle_model_list(available_models, current_model)
+                    continue
+                
+                if user_input.lower() == 'model':
+                    new_model, should_restart = handle_model_switch(available_models, current_model)
+                    if should_restart:
+                        current_model = new_model
+                        conversation_active = False
+                    continue
+                
+                if not user_input:
+                    continue
+                
+                # Stream response from agent
+                await stream_response(client, user_input)
+
+
+def main():
+    """Run interactive chat"""
+    try:
+        asyncio.run(interactive_chat())
+    except KeyboardInterrupt:
+        print("\n\n👋 Goodbye!")
+
+
+if __name__ == "__main__":
+    main()

cookbook/anthropic_agent_sdk/requirements.txt

@@ -0,0 +1,2 @@
+claude-agent-sdk
+httpx>=0.27.0

cookbook/benchmark/readme.md

@@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?':
 ```
 
 ## Support 
-**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
+**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.

cookbook/gollem_go_agent_framework/README.md

@@ -0,0 +1,119 @@
+# Gollem Go Agent Framework with LiteLLM
+
+A working example showing how to use [gollem](https://github.com/fugue-labs/gollem), a production-grade Go agent framework, with LiteLLM as a proxy gateway. This lets Go developers access 100+ LLM providers through a single proxy while keeping compile-time type safety for tools and structured output.
+
+## Quick Start
+
+### 1. Start LiteLLM Proxy
+
+```bash
+# Simple start with a single model
+litellm --model gpt-4o
+
+# Or with the example config for multi-provider access
+litellm --config proxy_config.yaml
+```
+
+### 2. Run the examples
+
+```bash
+# Install Go dependencies
+go mod tidy
+
+# Basic agent
+go run ./basic
+
+# Agent with type-safe tools
+go run ./tools
+
+# Streaming responses
+go run ./streaming
+```
+
+## Configuration
+
+The included `proxy_config.yaml` sets up three providers through LiteLLM:
+
+```yaml
+model_list:
+  - model_name: gpt-4o          # OpenAI
+  - model_name: claude-sonnet    # Anthropic
+  - model_name: gemini-pro       # Google Vertex AI
+```
+
+Switch providers in Go by changing a single string — no code changes needed:
+
+```go
+model := openai.NewLiteLLM("http://localhost:4000",
+    openai.WithModel("gpt-4o"),        // OpenAI
+    // openai.WithModel("claude-sonnet"),  // Anthropic
+    // openai.WithModel("gemini-pro"),     // Google
+)
+```
+
+## Examples
+
+### `basic/` — Basic Agent
+
+Connects gollem to LiteLLM and runs a simple prompt. Demonstrates the `NewLiteLLM` constructor and basic agent creation.
+
+### `tools/` — Type-Safe Tools
+
+Shows gollem's compile-time type-safe tool framework working through LiteLLM's tool-use passthrough. The tool parameters are Go structs with JSON tags — the schema is generated automatically at compile time.
+
+### `streaming/` — Streaming Responses
+
+Real-time token streaming using Go 1.23+ range-over-function iterators, proxied through LiteLLM's SSE passthrough.
+
+## How It Works
+
+Gollem's `openai.NewLiteLLM()` constructor creates an OpenAI-compatible provider pointed at your LiteLLM proxy. Since LiteLLM speaks the OpenAI API protocol, everything works out of the box:
+
+- **Chat completions** — standard request/response
+- **Tool use** — LiteLLM passes tool definitions and calls through transparently
+- **Streaming** — Server-Sent Events proxied through LiteLLM
+- **Structured output** — JSON schema response format works with supporting models
+
+```
+Go App (gollem) → LiteLLM Proxy → OpenAI / Anthropic / Google / ...
+```
+
+## Why Use This?
+
+- **Type-safe Go**: Compile-time type checking for tools, structured output, and agent configuration — no runtime surprises
+- **Single proxy, many models**: Switch between OpenAI, Anthropic, Google, and 100+ other providers by changing a model name string
+- **Zero-dependency core**: gollem's core has no external dependencies — just stdlib
+- **Single binary deployment**: `go build` produces one binary, no pip/venv/Docker needed
+- **Cost tracking & rate limiting**: LiteLLM handles cost tracking, rate limits, and fallbacks at the proxy layer
+
+## Environment Variables
+
+```bash
+# Required for providers you want to use (set in LiteLLM config or env)
+export OPENAI_API_KEY="sk-..."
+export ANTHROPIC_API_KEY="sk-ant-..."
+
+# Optional: point to a non-default LiteLLM proxy
+export LITELLM_PROXY_URL="http://localhost:4000"
+```
+
+## Troubleshooting
+
+**Connection errors?**
+- Make sure LiteLLM is running: `litellm --model gpt-4o`
+- Check the URL is correct (default: `http://localhost:4000`)
+
+**Model not found?**
+- Verify the model name matches what's configured in LiteLLM
+- Run `curl http://localhost:4000/models` to see available models
+
+**Tool calls not working?**
+- Ensure the underlying model supports tool use (GPT-4o, Claude, Gemini)
+- Check LiteLLM logs for any provider-specific errors
+
+## Learn More
+
+- [gollem GitHub](https://github.com/fugue-labs/gollem)
+- [gollem API Reference](https://pkg.go.dev/github.com/fugue-labs/gollem/core)
+- [LiteLLM Proxy Docs](https://docs.litellm.ai/docs/simple_proxy)
+- [LiteLLM Supported Models](https://docs.litellm.ai/docs/providers)

cookbook/gollem_go_agent_framework/basic/main.go

@@ -0,0 +1,41 @@
+// Basic gollem agent connected to a LiteLLM proxy.
+//
+// Usage:
+//
+//	litellm --model gpt-4o   # start proxy in another terminal
+//	go run ./basic
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	// Connect to LiteLLM proxy. NewLiteLLM creates an OpenAI-compatible
+	// provider pointed at the given URL.
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"), // any model name configured in LiteLLM
+	)
+
+	// Create and run a simple agent.
+	agent := core.NewAgent[string](model,
+		core.WithSystemPrompt[string]("You are a helpful assistant. Be concise."),
+	)
+
+	result, err := agent.Run(context.Background(), "Explain quantum computing in two sentences.")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(result.Output)
+}

cookbook/gollem_go_agent_framework/go.mod

@@ -0,0 +1,5 @@
+module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework
+
+go 1.25.1
+
+require github.com/fugue-labs/gollem v0.1.0

cookbook/gollem_go_agent_framework/go.sum

@@ -0,0 +1,2 @@
+github.com/fugue-labs/gollem v0.1.0 h1:QexYnvkb44QZFEljgAePqMIGZjgsbk0Y5GJ2jYYgfa8=
+github.com/fugue-labs/gollem v0.1.0/go.mod h1:htW1YO81uysSKVOkYJtxhGCFrzm+36HBFxEWuECoHKQ=

cookbook/gollem_go_agent_framework/proxy_config.yaml

@@ -0,0 +1,16 @@
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: claude-sonnet
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: gemini-pro
+    litellm_params:
+      model: vertex_ai/gemini-2.0-flash
+      vertex_project: my-project
+      vertex_location: us-central1

cookbook/gollem_go_agent_framework/streaming/main.go

@@ -0,0 +1,56 @@
+// Streaming responses from gollem through LiteLLM.
+//
+// Uses Go 1.23+ range-over-function iterators for real-time token
+// streaming via LiteLLM's SSE passthrough.
+//
+// Usage:
+//
+//	litellm --model gpt-4o
+//	go run ./streaming
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"),
+	)
+
+	agent := core.NewAgent[string](model)
+
+	// RunStream returns a streaming result that yields tokens as they arrive.
+	stream, err := agent.RunStream(context.Background(), "Write a haiku about distributed systems")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// StreamText yields text chunks in real-time.
+	// The boolean argument controls whether deltas (true) or accumulated
+	// text (false) is returned.
+	fmt.Print("Response: ")
+	for text, err := range stream.StreamText(true) {
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Print(text)
+	}
+	fmt.Println()
+
+	// After streaming completes, the final response is available.
+	resp := stream.Response()
+	fmt.Printf("\nTokens used: input=%d, output=%d\n",
+		resp.Usage.InputTokens, resp.Usage.OutputTokens)
+}

cookbook/gollem_go_agent_framework/tools/main.go

@@ -0,0 +1,64 @@
+// Gollem agent with type-safe tools through LiteLLM.
+//
+// The tool parameters are Go structs — gollem generates the JSON schema
+// automatically at compile time. LiteLLM passes tool definitions through
+// transparently to the underlying provider.
+//
+// Usage:
+//
+//	litellm --model gpt-4o
+//	go run ./tools
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+// WeatherParams defines the tool's input schema via struct tags.
+// The JSON schema is generated at compile time — no runtime reflection needed.
+type WeatherParams struct {
+	City string `json:"city" description:"City name to get weather for"`
+	Unit string `json:"unit,omitempty" description:"Temperature unit: celsius or fahrenheit"`
+}
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"),
+	)
+
+	// Define a type-safe tool. The function signature enforces correct types.
+	weatherTool := core.FuncTool[WeatherParams](
+		"get_weather",
+		"Get current weather for a city",
+		func(ctx context.Context, p WeatherParams) (string, error) {
+			unit := p.Unit
+			if unit == "" {
+				unit = "fahrenheit"
+			}
+			// In production, call a real weather API here.
+			return fmt.Sprintf("Weather in %s: 72°F (22°C), sunny", p.City), nil
+		},
+	)
+
+	agent := core.NewAgent[string](model,
+		core.WithTools[string](weatherTool),
+		core.WithSystemPrompt[string]("You are a helpful weather assistant. Use the get_weather tool to answer weather questions."),
+	)
+
+	result, err := agent.Run(context.Background(), "What's the weather like in San Francisco and Tokyo?")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(result.Output)
+}

cookbook/livekit_agent_sdk/README.md

@@ -0,0 +1,114 @@
+# LiveKit Voice Agent with LiteLLM Gateway
+
+Simple example showing how to use LiveKit's xAI realtime plugin with LiteLLM as a proxy. This lets you switch between xAI, OpenAI, and Azure realtime APIs without changing your code.
+
+## Quick Start
+
+### 1. Install dependencies
+
+```bash
+pip install livekit-agents[xai] websockets
+```
+
+### 2. Start LiteLLM proxy
+
+```bash
+# With xAI
+export XAI_API_KEY="your-xai-key"
+litellm --config config.yaml --port 4000
+```
+
+### 3. Run the voice agent
+
+```bash
+python main.py
+```
+
+Type your message and get a voice response from Grok!
+
+## Configuration
+
+Set these environment variables if needed:
+
+```bash
+export LITELLM_PROXY_URL="http://localhost:4000"
+export LITELLM_API_KEY="sk-1234"
+export LITELLM_MODEL="grok-voice-agent"
+```
+
+Or use the defaults - connects to `http://localhost:4000` by default.
+
+## Example Config File
+
+Create a `config.yaml` with your realtime models:
+
+```yaml
+model_list:
+  - model_name: grok-voice-agent
+    litellm_params:
+      model: xai/grok-2-vision-1212
+      api_key: os.environ/XAI_API_KEY
+    model_info:
+      mode: realtime
+
+  - model_name: openai-voice-agent
+    litellm_params:
+      model: gpt-4o-realtime-preview
+      api_key: os.environ/OPENAI_API_KEY
+    model_info:
+      mode: realtime
+
+general_settings:
+  master_key: sk-1234
+```
+
+Then start: `litellm --config config.yaml --port 4000`
+
+## How It Works
+
+LiveKit's xAI plugin connects through LiteLLM proxy by setting `base_url`:
+
+```python
+from livekit.plugins import xai
+
+model = xai.realtime.RealtimeModel(
+    voice="ara",
+    api_key="sk-1234",              # LiteLLM proxy key
+    base_url="http://localhost:4000", # Point to LiteLLM
+)
+```
+
+## Switching Providers
+
+Just change the model in your config - no code changes needed:
+
+**xAI Grok:**
+```yaml
+model: xai/grok-2-vision-1212
+```
+
+**OpenAI:**
+```yaml
+model: gpt-4o-realtime-preview
+```
+
+**Azure OpenAI:**
+```yaml
+model: azure/gpt-4o-realtime-preview
+api_base: https://your-endpoint.openai.azure.com/
+```
+
+## Why Use LiteLLM?
+
+- ✅ **Switch providers** without changing agent code
+- ✅ **Cost tracking** across all voice sessions
+- ✅ **Rate limiting** and budgets
+- ✅ **Load balancing** across multiple API keys
+- ✅ **Fallbacks** to backup models
+
+## Learn More
+
+- [LiveKit xAI Realtime Tutorial](/docs/tutorials/livekit_xai_realtime)
+- [xAI Realtime Docs](/docs/providers/xai_realtime)
+- [LiveKit Agents Documentation](https://docs.livekit.io/agents/)
+- [LiteLLM Realtime API](/docs/realtime)

cookbook/livekit_agent_sdk/config.example.yaml

@@ -0,0 +1,21 @@
+model_list:
+  - model_name: grok-voice-agent
+    litellm_params:
+      model: xai/grok-2-vision-1212
+      api_key: os.environ/XAI_API_KEY
+    model_info:
+      mode: realtime
+
+  - model_name: openai-voice-agent
+    litellm_params:
+      model: gpt-4o-realtime-preview
+      api_key: os.environ/OPENAI_API_KEY
+    model_info:
+      mode: realtime
+
+litellm_settings:
+  drop_params: True
+  telemetry: False
+
+general_settings:
+  master_key: sk-1234  # Change this to a secure key

cookbook/livekit_agent_sdk/main.py

@@ -0,0 +1,112 @@
+"""
+Simple xAI Voice Agent using LiveKit SDK with LiteLLM Gateway
+
+This example shows how to use LiveKit's xAI realtime plugin through LiteLLM proxy.
+LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI, 
+and Azure realtime APIs without changing your agent code.
+"""
+import asyncio
+import json
+import os
+import websockets
+
+# Configuration
+PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
+API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
+MODEL = os.getenv("LITELLM_MODEL", "grok-voice-agent")
+
+
+async def run_voice_agent():
+    """
+    Simple voice agent that:
+    1. Connects to xAI realtime API through LiteLLM proxy
+    2. Sends a user message
+    3. Streams back the response
+    """
+    
+    url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}"
+    headers = {"Authorization": f"Bearer {API_KEY}"}
+    
+    print(f"🎙️  Connecting to voice agent...")
+    print(f"   Model: {MODEL}")
+    print(f"   Proxy: {PROXY_URL}")
+    print()
+    
+    async with websockets.connect(url, additional_headers=headers) as ws:
+        # Receive initial connection event
+        initial = json.loads(await ws.recv())
+        print(f"✅ Connected! Event: {initial['type']}\n")
+        
+        # Get user input
+        user_message = input("💬 Your message: ").strip()
+        if not user_message:
+            user_message = "Tell me a fun fact about AI!"
+        
+        print(f"\n🤖 Sending to {MODEL}...\n")
+        
+        # Send user message
+        await ws.send(json.dumps({
+            "type": "conversation.item.create",
+            "item": {
+                "type": "message",
+                "role": "user",
+                "content": [{"type": "input_text", "text": user_message}]
+            }
+        }))
+        
+        # Request response
+        await ws.send(json.dumps({
+            "type": "response.create",
+            "response": {"modalities": ["text", "audio"]}
+        }))
+        
+        # Stream response
+        print("🎤 Response: ", end='', flush=True)
+        transcript = []
+        
+        try:
+            while True:
+                msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
+                event = json.loads(msg)
+                
+                # Capture transcript deltas
+                if event['type'] == 'response.output_audio_transcript.delta':
+                    delta = event.get('delta', '')
+                    if delta:
+                        print(delta, end='', flush=True)
+                        transcript.append(delta)
+                
+                # Done when response completes
+                elif event['type'] == 'response.done':
+                    break
+        
+        except asyncio.TimeoutError:
+            pass
+        
+        print("\n")
+        
+        if transcript:
+            print(f"✅ Complete response: {''.join(transcript)}")
+        
+        await ws.close()
+
+
+def main():
+    """Run the voice agent"""
+    print("=" * 70)
+    print("LiveKit xAI Voice Agent via LiteLLM Proxy")
+    print("=" * 70)
+    print()
+    
+    try:
+        asyncio.run(run_voice_agent())
+    except KeyboardInterrupt:
+        print("\n\n👋 Goodbye!")
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        print("\nMake sure LiteLLM proxy is running:")
+        print(f"  litellm --config config.yaml --port 4000")
+
+
+if __name__ == "__main__":
+    main()

cookbook/livekit_agent_sdk/requirements.txt

@@ -0,0 +1,2 @@
+livekit-agents[xai]>=1.3.12
+websockets>=15.0.1

cookbook/mock_prompt_management_server/README.md

@@ -0,0 +1,293 @@
+# Mock Prompt Management Server
+
+A reference implementation of the [LiteLLM Generic Prompt Management API](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api).
+
+This FastAPI server demonstrates how to build a prompt management API that integrates with LiteLLM without requiring a PR to the LiteLLM repository.
+
+## Quick Start
+
+### 1. Install Dependencies
+
+```bash
+pip install fastapi uvicorn pydantic
+```
+
+### 2. Start the Server
+
+```bash
+python mock_prompt_management_server.py
+```
+
+The server will start on `http://localhost:8080`
+
+### 3. Test the Endpoint
+
+```bash
+# Get a prompt
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
+
+# Get a prompt with authentication
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" \
+  -H "Authorization: Bearer test-token-12345"
+
+# List all prompts
+curl "http://localhost:8080/prompts"
+
+# Get prompt variables
+curl "http://localhost:8080/prompts/hello-world-prompt/variables"
+```
+
+## Using with LiteLLM
+
+### Configuration
+
+Create a `config.yaml` file:
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+
+prompts:
+  - prompt_id: "hello-world-prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      api_base: http://localhost:8080
+      api_key: test-token-12345
+```
+
+### Start LiteLLM Proxy
+
+```bash
+litellm --config config.yaml
+```
+
+### Make a Request
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "prompt_id": "hello-world-prompt",
+    "prompt_variables": {
+      "domain": "data science",
+      "task": "analyzing customer behavior"
+    },
+    "messages": [
+      {"role": "user", "content": "Please help me get started"}
+    ]
+  }'
+```
+
+## Available Prompts
+
+The server includes several example prompts:
+
+| Prompt ID | Description | Variables |
+|-----------|-------------|-----------|
+| `hello-world-prompt` | Basic helpful assistant | `domain`, `task` |
+| `code-review-prompt` | Code review assistant | `years_experience`, `language`, `code` |
+| `customer-support-prompt` | Customer support agent | `company_name`, `customer_message` |
+| `data-analysis-prompt` | Data analysis expert | `analysis_type`, `dataset_name`, `data` |
+| `creative-writing-prompt` | Creative writing assistant | `genre`, `length`, `topic` |
+
+## Authentication
+
+The server supports optional Bearer token authentication. Valid tokens for testing:
+
+- `test-token-12345`
+- `dev-token-67890`
+- `prod-token-abcdef`
+
+If no `Authorization` header is provided, requests are allowed (for testing purposes).
+
+## API Endpoints
+
+### LiteLLM Spec Endpoints
+
+#### `GET /beta/litellm_prompt_management`
+
+Get a prompt by ID (required by LiteLLM).
+
+**Query Parameters:**
+- `prompt_id` (required): The prompt ID
+- `project_name` (optional): Project filter
+- `slug` (optional): Slug filter
+- `version` (optional): Version filter
+
+**Response:**
+```json
+{
+  "prompt_id": "hello-world-prompt",
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant specialized in {domain}."
+    },
+    {
+      "role": "user",
+      "content": "Help me with: {task}"
+    }
+  ],
+  "prompt_template_model": "gpt-4",
+  "prompt_template_optional_params": {
+    "temperature": 0.7,
+    "max_tokens": 500
+  }
+}
+```
+
+### Convenience Endpoints (Not in LiteLLM Spec)
+
+#### `GET /health`
+
+Health check endpoint.
+
+#### `GET /prompts`
+
+List all available prompts.
+
+#### `GET /prompts/{prompt_id}/variables`
+
+Get all variables used in a prompt template.
+
+#### `POST /prompts`
+
+Create a new prompt (in-memory only, for testing).
+
+## Example: Full Integration Test
+
+### 1. Start the Mock Server
+
+```bash
+python mock_prompt_management_server.py
+```
+
+### 2. Test with Python
+
+```python
+from litellm import completion
+
+# The completion will:
+# 1. Fetch the prompt from your API
+# 2. Replace {domain} with "machine learning"
+# 3. Replace {task} with "building a recommendation system"
+# 4. Merge with your messages
+# 5. Use the model and params from the prompt
+
+response = completion(
+    model="gpt-4",
+    prompt_id="hello-world-prompt",
+    prompt_variables={
+        "domain": "machine learning",
+        "task": "building a recommendation system"
+    },
+    messages=[
+        {"role": "user", "content": "I have user behavior data from the past year."}
+    ],
+    # Configure the generic prompt manager
+    generic_prompt_config={
+        "api_base": "http://localhost:8080",
+        "api_key": "test-token-12345",
+    }
+)
+
+print(response.choices[0].message.content)
+```
+
+## Customization
+
+### Adding New Prompts
+
+Edit the `PROMPTS_DB` dictionary in `mock_prompt_management_server.py`:
+
+```python
+PROMPTS_DB = {
+    "my-custom-prompt": {
+        "prompt_id": "my-custom-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a {role}."
+            },
+            {
+                "role": "user",
+                "content": "{user_input}"
+            }
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.8,
+            "max_tokens": 1000
+        }
+    }
+}
+```
+
+### Using a Database
+
+Replace the `PROMPTS_DB` dictionary with database queries:
+
+```python
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(prompt_id: str):
+    # Fetch from database
+    prompt = await db.prompts.find_one({"prompt_id": prompt_id})
+    
+    if not prompt:
+        raise HTTPException(status_code=404, detail="Prompt not found")
+    
+    return PromptResponse(**prompt)
+```
+
+### Adding Access Control
+
+Use the custom query parameters for access control:
+
+```python
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(
+    prompt_id: str,
+    project_name: Optional[str] = None,
+    user_id: Optional[str] = None,
+    authorization: Optional[str] = Header(None)
+):
+    token = verify_api_key(authorization)
+    
+    # Check if user has access to this project
+    if not has_project_access(token, project_name):
+        raise HTTPException(status_code=403, detail="Access denied")
+    
+    # Fetch and return prompt
+    ...
+```
+
+## Production Considerations
+
+Before deploying to production:
+
+1. **Use a real database** instead of in-memory storage
+2. **Implement proper authentication** with JWT tokens or API keys
+3. **Add rate limiting** to prevent abuse
+4. **Use HTTPS** for encrypted communication
+5. **Add logging and monitoring** for observability
+6. **Implement caching** for frequently accessed prompts
+7. **Add versioning** for prompt management
+8. **Implement access control** based on teams/users
+9. **Add input validation** for all parameters
+10. **Use environment variables** for configuration
+
+## Related Documentation
+
+- [Generic Prompt Management API Documentation](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api)
+- [LiteLLM Prompt Management](https://docs.litellm.ai/docs/proxy/prompt_management)
+- [Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api)
+
+## Questions?
+
+This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm).
+

cookbook/mock_prompt_management_server/mock_prompt_management_server.py

@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+"""
+Mock Prompt Management API Server
+
+This is a FastAPI server that implements the LiteLLM Generic Prompt Management API
+for testing and demonstration purposes.
+
+Usage:
+    python mock_prompt_management_server.py
+
+The server will start on http://localhost:8080
+
+Test the endpoint:
+    curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
+"""
+
+import os
+import json
+from typing import Any, Dict, List, Optional
+
+from fastapi import FastAPI, HTTPException, Header, Query, status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+# ============================================================================
+# Response Models
+# ============================================================================
+
+
+class MessageContent(BaseModel):
+    """A single message in the prompt template"""
+
+    role: str = Field(..., description="Message role (system, user, assistant)")
+    content: str = Field(
+        ..., description="Message content with optional {variable} placeholders"
+    )
+
+
+class PromptResponse(BaseModel):
+    """Response format for the prompt management API"""
+
+    prompt_id: str = Field(..., description="The ID of the prompt")
+    prompt_template: List[MessageContent] = Field(
+        ..., description="Array of messages in OpenAI format"
+    )
+    prompt_template_model: Optional[str] = Field(
+        None, description="Optional model to use for this prompt"
+    )
+    prompt_template_optional_params: Optional[Dict[str, Any]] = Field(
+        None, description="Optional parameters like temperature, max_tokens, etc."
+    )
+
+
+# ============================================================================
+# Mock Prompt Database
+# ============================================================================
+
+PROMPTS_DB = {
+    "hello-world-prompt": {
+        "prompt_id": "hello-world-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a helpful assistant specialized in {domain}.",
+            },
+            {"role": "user", "content": "Help me with: {task}"},
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {"temperature": 0.7, "max_tokens": 500},
+    },
+    "code-review-prompt": {
+        "prompt_id": "code-review-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are an expert code reviewer with {years_experience} years of experience in {language}.",
+            },
+            {
+                "role": "user",
+                "content": "Please review the following code for bugs, security issues, and best practices:\n\n{code}",
+            },
+        ],
+        "prompt_template_model": "gpt-4-turbo",
+        "prompt_template_optional_params": {
+            "temperature": 0.3,
+            "max_tokens": 1500,
+        },
+    },
+    "customer-support-prompt": {
+        "prompt_id": "customer-support-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a friendly customer support agent for {company_name}. Always be professional, empathetic, and solution-oriented.",
+            },
+            {
+                "role": "user",
+                "content": "Customer inquiry: {customer_message}",
+            },
+        ],
+        "prompt_template_model": "gpt-3.5-turbo",
+        "prompt_template_optional_params": {
+            "temperature": 0.8,
+            "max_tokens": 800,
+            "top_p": 0.9,
+        },
+    },
+    "data-analysis-prompt": {
+        "prompt_id": "data-analysis-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a data scientist expert in {analysis_type} analysis.",
+            },
+            {
+                "role": "user",
+                "content": "Analyze the following data and provide insights:\n\nDataset: {dataset_name}\nData: {data}",
+            },
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.5,
+            "max_tokens": 2000,
+        },
+    },
+    "creative-writing-prompt": {
+        "prompt_id": "creative-writing-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a creative writer specializing in {genre} fiction.",
+            },
+            {
+                "role": "user",
+                "content": "Write a {length} story about: {topic}",
+            },
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.9,
+            "max_tokens": 3000,
+            "top_p": 0.95,
+        },
+    },
+}
+
+# Valid API tokens for authentication (in production, use a secure token store)
+VALID_API_TOKENS = {
+    "test-token-12345",
+    "dev-token-67890",
+    "prod-token-abcdef",
+}
+
+# ============================================================================
+# FastAPI App
+# ============================================================================
+
+app = FastAPI(
+    title="Mock Prompt Management API",
+    description="A mock server implementing the LiteLLM Generic Prompt Management API",
+    version="1.0.0",
+)
+
+
+def verify_api_key(authorization: Optional[str] = Header(None)) -> bool:
+    """
+    Verify the API key from the Authorization header.
+
+    Args:
+        authorization: Authorization header (Bearer token)
+
+    Returns:
+        True if valid, raises HTTPException if invalid
+    """
+    if authorization is None:
+        # Allow requests without authentication for testing
+        return True
+
+    # Extract token from "Bearer <token>"
+    if not authorization.startswith("Bearer "):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid authorization header format. Expected 'Bearer <token>'",
+        )
+
+    token = authorization.replace("Bearer ", "").strip()
+
+    if token not in VALID_API_TOKENS:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid API key",
+        )
+
+    return True
+
+
+@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
+async def get_prompt(
+    prompt_id: str = Query(..., description="The ID of the prompt to fetch"),
+    project_name: Optional[str] = Query(
+        None, description="Optional project name filter"
+    ),
+    slug: Optional[str] = Query(None, description="Optional slug filter"),
+    version: Optional[str] = Query(None, description="Optional version filter"),
+    authorization: Optional[str] = Header(None),
+) -> PromptResponse:
+    """
+    Get a prompt by ID with optional filtering.
+
+    This endpoint implements the LiteLLM Generic Prompt Management API specification.
+
+    Args:
+        prompt_id: The ID of the prompt to fetch
+        project_name: Optional project name for filtering
+        slug: Optional slug for filtering
+        version: Optional version for filtering
+        authorization: Optional Bearer token for authentication
+
+    Returns:
+        PromptResponse with the prompt template and configuration
+
+    Raises:
+        HTTPException: 401 if authentication fails, 404 if prompt not found
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    # Log the request parameters (useful for debugging)
+    print(f"Fetching prompt: {prompt_id}")
+    if project_name:
+        print(f"  Project: {project_name}")
+    if slug:
+        print(f"  Slug: {slug}")
+    if version:
+        print(f"  Version: {version}")
+
+    # Check if prompt exists
+    if prompt_id not in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Prompt '{prompt_id}' not found. Available prompts: {list(PROMPTS_DB.keys())}",
+        )
+
+    # Get the prompt from the database
+    prompt_data = PROMPTS_DB[prompt_id]
+
+    # Optional: Apply filtering based on project_name, slug, or version
+    # In a real implementation, you might use these to filter prompts by access control
+    # or to fetch specific versions from your database
+
+    return PromptResponse(**prompt_data)
+
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint"""
+    return {
+        "status": "healthy",
+        "service": "mock-prompt-management-api",
+        "version": "1.0.0",
+    }
+
+
+@app.get("/prompts")
+async def list_prompts(authorization: Optional[str] = Header(None)):
+    """
+    List all available prompts.
+
+    This is a convenience endpoint (not part of the LiteLLM spec) for
+    discovering available prompts.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    prompts_list = [
+        {
+            "prompt_id": pid,
+            "model": p.get("prompt_template_model"),
+            "has_variables": any(
+                "{" in msg.get("content", "") for msg in p.get("prompt_template", [])
+            ),
+        }
+        for pid, p in PROMPTS_DB.items()
+    ]
+
+    return {"prompts": prompts_list, "total": len(prompts_list)}
+
+
+@app.get("/prompts/{prompt_id}/variables")
+async def get_prompt_variables(
+    prompt_id: str, authorization: Optional[str] = Header(None)
+):
+    """
+    Get all variables in a prompt template.
+
+    This is a convenience endpoint (not part of the LiteLLM spec) for
+    discovering what variables a prompt expects.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    if prompt_id not in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Prompt '{prompt_id}' not found",
+        )
+
+    prompt_data = PROMPTS_DB[prompt_id]
+    variables = set()
+
+    # Extract variables from the prompt template
+    import re
+
+    for message in prompt_data["prompt_template"]:
+        content = message.get("content", "")
+        # Find all {variable} patterns
+        found_vars = re.findall(r"\{(\w+)\}", content)
+        variables.update(found_vars)
+
+    return {
+        "prompt_id": prompt_id,
+        "variables": sorted(list(variables)),
+        "example_usage": {
+            "prompt_id": prompt_id,
+            "prompt_variables": {var: f"<{var}_value>" for var in variables},
+        },
+    }
+
+
+@app.post("/prompts")
+async def create_prompt(
+    prompt: PromptResponse, authorization: Optional[str] = Header(None)
+):
+    """
+    Create a new prompt (convenience endpoint for testing).
+
+    This is NOT part of the LiteLLM spec - it's just for testing purposes.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    if prompt.prompt_id in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"Prompt '{prompt.prompt_id}' already exists",
+        )
+
+    PROMPTS_DB[prompt.prompt_id] = prompt.dict()
+
+    return {
+        "status": "created",
+        "prompt_id": prompt.prompt_id,
+        "message": "Prompt created successfully (in-memory only)",
+    }
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+if __name__ == "__main__":
+    import uvicorn
+
+    print("=" * 70)
+    print("Mock Prompt Management API Server")
+    print("=" * 70)
+    print(f"\nStarting server on http://localhost:8080")
+    print(f"\nAvailable prompts: {len(PROMPTS_DB)}")
+    for prompt_id in PROMPTS_DB.keys():
+        print(f"  - {prompt_id}")
+    print(f"\nValid API tokens: {len(VALID_API_TOKENS)}")
+    print("  - test-token-12345")
+    print("  - dev-token-67890")
+    print("  - prod-token-abcdef")
+    print("\nEndpoints:")
+    print("  GET  /beta/litellm_prompt_management?prompt_id=<id>  (LiteLLM spec)")
+    print("  GET  /health                                          (health check)")
+    print("  GET  /prompts                                         (list all prompts)")
+    print(
+        "  GET  /prompts/{id}/variables                          (get prompt variables)"
+    )
+    print("  POST /prompts                                         (create prompt)")
+    print("\nExample usage:")
+    print(
+        '  curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"'
+    )
+    print("\nPress CTRL+C to stop the server")
+    print("=" * 70)
+
+    uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info")

cookbook/nova_sonic_realtime.py

@@ -0,0 +1,288 @@
+"""
+Client script to test Nova Sonic realtime API through LiteLLM proxy.
+
+This script connects to LiteLLM proxy's realtime endpoint and enables
+speech-to-speech conversation with Bedrock Nova Sonic.
+
+Prerequisites:
+- LiteLLM proxy running with Bedrock configured
+- pyaudio installed: pip install pyaudio
+- websockets installed: pip install websockets
+
+Usage:
+    python nova_sonic_realtime.py
+"""
+
+import asyncio
+import base64
+import json
+import os
+import pyaudio
+import websockets
+from typing import Optional
+
+# Bounded queue size for audio chunks (configurable via env to avoid unbounded memory)
+AUDIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 10_000))
+
+# Audio configuration (matching Nova Sonic requirements)
+INPUT_SAMPLE_RATE = 16000  # Nova Sonic expects 16kHz input
+OUTPUT_SAMPLE_RATE = 24000  # Nova Sonic outputs 24kHz
+CHANNELS = 1
+FORMAT = pyaudio.paInt16
+CHUNK_SIZE = 1024
+
+# LiteLLM proxy configuration
+LITELLM_PROXY_URL = "ws://localhost:4000/v1/realtime?model=bedrock-sonic"
+LITELLM_API_KEY = "sk-12345"  # Your LiteLLM API key
+
+
+class RealtimeClient:
+    """Client for LiteLLM realtime API with audio support."""
+
+    def __init__(self, url: str, api_key: str):
+        self.url = url
+        self.api_key = api_key
+        self.ws: Optional[websockets.WebSocketClientProtocol] = None
+        self.is_active = False
+        self.audio_queue = asyncio.Queue(maxsize=AUDIO_QUEUE_MAXSIZE)
+        self.pyaudio = pyaudio.PyAudio()
+        self.input_stream = None
+        self.output_stream = None
+
+    async def connect(self):
+        """Connect to LiteLLM proxy realtime endpoint."""
+        print(f"Connecting to {self.url}...")
+        
+        headers = {}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        
+        self.ws = await websockets.connect(
+            self.url,
+            additional_headers=headers,
+            max_size=10 * 1024 * 1024,  # 10MB max message size
+        )
+        self.is_active = True
+        print("✓ Connected to LiteLLM proxy")
+
+    async def send_session_update(self):
+        """Send session configuration."""
+        session_update = {
+            "type": "session.update",
+            "session": {
+                "instructions": "You are a friendly assistant. Keep your responses short and conversational.",
+                "voice": "matthew",
+                "temperature": 0.8,
+                "max_response_output_tokens": 1024,
+                "modalities": ["text", "audio"],
+                "input_audio_format": "pcm16",
+                "output_audio_format": "pcm16",
+                "turn_detection": {
+                    "type": "server_vad",
+                    "threshold": 0.5,
+                    "prefix_padding_ms": 300,
+                    "silence_duration_ms": 500,
+                },
+            },
+        }
+        await self.ws.send(json.dumps(session_update))
+        print("✓ Session configuration sent")
+
+    async def receive_messages(self):
+        """Receive and process messages from the server."""
+        try:
+            async for message in self.ws:
+                if not self.is_active:
+                    break
+
+                try:
+                    data = json.loads(message)
+                    event_type = data.get("type")
+
+                    if event_type == "session.created":
+                        print(f"✓ Session created: {data.get('session', {}).get('id')}")
+
+                    elif event_type == "response.created":
+                        print("🤖 Assistant is responding...")
+
+                    elif event_type == "response.text.delta":
+                        # Print text transcription
+                        delta = data.get("delta", "")
+                        print(delta, end="", flush=True)
+
+                    elif event_type == "response.audio.delta":
+                        # Queue audio for playback
+                        audio_b64 = data.get("delta", "")
+                        if audio_b64:
+                            audio_bytes = base64.b64decode(audio_b64)
+                            await self.audio_queue.put(audio_bytes)
+
+                    elif event_type == "response.text.done":
+                        print()  # New line after text
+
+                    elif event_type == "response.done":
+                        print("✓ Response complete")
+
+                    elif event_type == "error":
+                        print(f"❌ Error: {data.get('error', {})}")
+
+                    else:
+                        # Debug: print other event types
+                        print(f"[{event_type}]", end=" ")
+
+                except json.JSONDecodeError:
+                    print(f"Failed to parse message: {message[:100]}")
+
+        except websockets.exceptions.ConnectionClosed:
+            print("\n✗ Connection closed")
+        except Exception as e:
+            print(f"\n✗ Error receiving messages: {e}")
+        finally:
+            self.is_active = False
+
+    async def send_audio_chunk(self, audio_bytes: bytes):
+        """Send audio chunk to server."""
+        if not self.is_active or not self.ws:
+            return
+
+        audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
+        message = {
+            "type": "input_audio_buffer.append",
+            "audio": audio_b64,
+        }
+        await self.ws.send(json.dumps(message))
+
+    async def commit_audio_buffer(self):
+        """Commit the audio buffer to trigger processing."""
+        if not self.is_active or not self.ws:
+            return
+
+        message = {"type": "input_audio_buffer.commit"}
+        await self.ws.send(json.dumps(message))
+
+    async def capture_audio(self):
+        """Capture audio from microphone and send to server."""
+        print("\n🎤 Starting audio capture...")
+        print("Speak into your microphone. Press Ctrl+C to stop.\n")
+
+        self.input_stream = self.pyaudio.open(
+            format=FORMAT,
+            channels=CHANNELS,
+            rate=INPUT_SAMPLE_RATE,
+            input=True,
+            frames_per_buffer=CHUNK_SIZE,
+        )
+
+        try:
+            while self.is_active:
+                audio_data = self.input_stream.read(CHUNK_SIZE, exception_on_overflow=False)
+                await self.send_audio_chunk(audio_data)
+                await asyncio.sleep(0.01)  # Small delay to prevent overwhelming
+        except Exception as e:
+            print(f"Error capturing audio: {e}")
+        finally:
+            if self.input_stream:
+                self.input_stream.stop_stream()
+                self.input_stream.close()
+
+    async def play_audio(self):
+        """Play audio responses from the server."""
+        print("🔊 Starting audio playback...")
+
+        self.output_stream = self.pyaudio.open(
+            format=FORMAT,
+            channels=CHANNELS,
+            rate=OUTPUT_SAMPLE_RATE,
+            output=True,
+            frames_per_buffer=CHUNK_SIZE,
+        )
+
+        try:
+            while self.is_active:
+                try:
+                    audio_data = await asyncio.wait_for(
+                        self.audio_queue.get(), timeout=0.1
+                    )
+                    if audio_data:
+                        self.output_stream.write(audio_data)
+                except asyncio.TimeoutError:
+                    continue
+        except Exception as e:
+            print(f"Error playing audio: {e}")
+        finally:
+            if self.output_stream:
+                self.output_stream.stop_stream()
+                self.output_stream.close()
+
+    async def close(self):
+        """Close the connection and cleanup."""
+        self.is_active = False
+
+        if self.ws:
+            await self.ws.close()
+
+        if self.input_stream:
+            self.input_stream.stop_stream()
+            self.input_stream.close()
+
+        if self.output_stream:
+            self.output_stream.stop_stream()
+            self.output_stream.close()
+
+        self.pyaudio.terminate()
+        print("\n✓ Connection closed")
+
+
+async def main():
+    """Main function to run the realtime client."""
+    print("=" * 80)
+    print("Bedrock Nova Sonic Realtime Client")
+    print("=" * 80)
+    print()
+
+    client = RealtimeClient(LITELLM_PROXY_URL, LITELLM_API_KEY)
+
+    try:
+        # Connect to server
+        await client.connect()
+
+        # Send session configuration
+        await client.send_session_update()
+
+        # Wait a moment for session to be established
+        await asyncio.sleep(0.5)
+
+        # Start tasks
+        receive_task = asyncio.create_task(client.receive_messages())
+        capture_task = asyncio.create_task(client.capture_audio())
+        playback_task = asyncio.create_task(client.play_audio())
+
+        # Wait for user to interrupt
+        await asyncio.gather(
+            receive_task,
+            capture_task,
+            playback_task,
+            return_exceptions=True,
+        )
+
+    except KeyboardInterrupt:
+        print("\n\n⚠ Interrupted by user")
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        import traceback
+        traceback.print_exc()
+    finally:
+        await client.close()
+
+
+if __name__ == "__main__":
+    print("\nMake sure:")
+    print("1. LiteLLM proxy is running on port 4000")
+    print("2. Bedrock is configured in proxy_server_config.yaml")
+    print("3. AWS credentials are set")
+    print()
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\n\nGoodbye!")

deploy/Dockerfile.ghcr_base

@@ -8,7 +8,8 @@ WORKDIR /app
 COPY config.yaml .
 
 # Make sure your docker/entrypoint.sh is executable
-RUN chmod +x docker/entrypoint.sh
+# Convert Windows line endings to Unix
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
 
 # Expose the necessary port
 EXPOSE 4000/tcp

deploy/charts/litellm-helm/Chart.yaml

@@ -18,13 +18,17 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.10
+version: 1.1.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: v1.50.2
+appVersion: v1.80.12
+
+annotations:
+  org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
+  org.opencontainers.image.url: "https://docs.litellm.ai/"
 
 dependencies:
   - name: "postgresql"

deploy/charts/litellm-helm/README.md

@@ -36,6 +36,10 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `serviceAccount.create`     | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API.                                                                                   | `false`                   |
 | `service.type`              | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                                                                              | `ClusterIP`               |
 | `service.port`              | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on.                                                                                                                          | `4000`                    |
+| `livenessProbe.*`           | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                  | See `values.yaml`         |
+| `readinessProbe.*`          | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                 | See `values.yaml`         |
+| `startupProbe.*`            | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                   | See `values.yaml`         |
+| `resources.*`               | CPU/memory requests and limits for the LiteLLM container.                                                                                                                                                                                       | `{}`                      |
 | `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                                                                  | `""`                      |
 | `ingress.labels`            | Additional labels for the Ingress resource                                                                                                                                                                                                    | `{}`                      |
 | `ingress.*`                 | See [values.yaml](./values.yaml) for example settings                                                                                                                                                                                         | N/A                       |

deploy/charts/litellm-helm/templates/_helpers.tpl

@@ -61,6 +61,20 @@ Create the name of the service account to use
 {{- end }}
 {{- end }}
 
+{{/*
+Create the service account name used by migration jobs.
+When Helm hooks are enabled, pre-install/pre-upgrade hooks run before normal resources.
+If this chart is creating the ServiceAccount, it is not yet available for the hook job,
+so fall back to "default" (or an explicit override) to avoid a cyclic dependency.
+*/}}
+{{- define "litellm.migrationServiceAccountName" -}}
+{{- if and .Values.migrationJob.hooks.helm.enabled .Values.serviceAccount.create }}
+{{- default "default" .Values.migrationJob.serviceAccountName }}
+{{- else }}
+{{- include "litellm.serviceAccountName" . }}
+{{- end }}
+{{- end }}
+
 {{/*
 Get redis service name
 */}}

deploy/charts/litellm-helm/templates/configmap-litellm.yaml

@@ -6,4 +6,4 @@ metadata:
 data:
   config.yaml: |
 {{ .Values.proxy_config | toYaml | indent 6 }}
-{{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -10,12 +10,19 @@ metadata:
     {{- toYaml .Values.deploymentLabels | nindent 4 }}
     {{- end }}
 spec:
-  {{- if not .Values.autoscaling.enabled }}
+  {{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
   replicas: {{ .Values.replicaCount }}
   {{- end }}
+  {{- with .Values.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.selectorLabels" . | nindent 6 }}
+  {{- if .Values.deploymentMinReadySeconds }}
+  minReadySeconds: {{ .Values.deploymentMinReadySeconds }}
+  {{- end }}
   template:
     metadata:
       annotations:
@@ -38,6 +45,10 @@ spec:
       serviceAccountName: {{ include "litellm.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- with .Values.extraInitContainers }}
+      initContainers:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       containers:
         - name: {{ include "litellm.name" . }}
           securityContext:
@@ -154,23 +165,37 @@ spec:
             {{- end }}
           livenessProbe:
             httpGet:
-              path: /health/liveliness
+              path: {{ .Values.livenessProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+            initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
           readinessProbe:
             httpGet:
-              path: /health/readiness
+              path: {{ .Values.readinessProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+            initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           startupProbe:
             httpGet:
-              path: /health/readiness
+              path: {{ .Values.startupProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
-            failureThreshold: 30
-            periodSeconds: 10
+            initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startupProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.startupProbe.successThreshold }}
+            failureThreshold: {{ .Values.startupProbe.failureThreshold }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
             - name: litellm-config
-              mountPath: /etc/litellm/
+              mountPath: /etc/litellm/config.yaml
+              subPath: config.yaml
           {{ if .Values.securityContext.readOnlyRootFilesystem }}
             - name: tmp
               mountPath: /tmp
@@ -182,6 +207,10 @@ spec:
           {{- with .Values.volumeMounts }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.lifecycle }}
+          lifecycle:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
       {{- with .Values.extraContainers }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
@@ -226,4 +255,4 @@ spec:
       {{- if .Values.topologySpreadConstraints }}
       topologySpreadConstraints:
         {{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
-      {{- end }}
+      {{- end }}

deploy/charts/litellm-helm/templates/keda.yaml

@@ -0,0 +1,37 @@
+{{- if and .Values.keda.enabled (not .Values.autoscaling.enabled) }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: {{ include "litellm.fullname" . }}
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
+  {{- if .Values.keda.scaledObject.annotations }}
+  annotations: {{ toYaml .Values.keda.scaledObject.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  scaleTargetRef:
+    name: {{ include "litellm.fullname" . }}
+  pollingInterval: {{ .Values.keda.pollingInterval }}
+  cooldownPeriod: {{ .Values.keda.cooldownPeriod }}
+  minReplicaCount: {{ .Values.keda.minReplicas }}
+  maxReplicaCount: {{ .Values.keda.maxReplicas }}
+{{- with .Values.keda.fallback }}
+  fallback:
+    failureThreshold: {{ .failureThreshold | default 3 }}
+    replicas: {{ .replicas | default $.Values.keda.maxReplicas }}
+{{- end }}
+  triggers:
+{{- with .Values.keda.triggers }}
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+  advanced:
+    restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }}
+{{- if .Values.keda.behavior }}
+    horizontalPodAutoscalerConfig:
+      behavior:
+{{- with .Values.keda.behavior }}
+{{- toYaml . | nindent 8 }}
+{{- end }}
+
+{{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -34,7 +34,11 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+      serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
+      {{- with .Values.migrationJob.extraInitContainers }}
+      initContainers:
+        {{- toYaml . | nindent 8 }}
+      {{- end }} 
       containers:
         - name: prisma-migrations
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}"

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -136,4 +136,186 @@ tests:
           path: spec.template.spec.containers[0].volumeMounts
           content:
             name: litellm-config
-            mountPath: /etc/litellm/
+            mountPath: /etc/litellm/config.yaml
+            subPath: config.yaml
+  - it: should work with lifecycle hooks
+    template: deployment.yaml
+    set:
+      lifecycle:
+        preStop:
+          exec:
+            command:
+              - /bin/sh
+              - -c
+              - echo "Container stopping"
+    asserts:
+      - exists:
+          path: spec.template.spec.containers[0].lifecycle
+      - equal:
+          path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[0]
+          value: /bin/sh
+      - equal:
+          path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[1]
+          value: -c
+      - equal:
+          path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
+          value: echo "Container stopping"
+  - it: should render background health check settings from proxy_config.general_settings
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.general_settings.background_health_checks: true
+      proxy_config.general_settings.health_check_interval: 240
+      proxy_config.general_settings.health_check_concurrency: 16
+      proxy_config.general_settings.health_check_details: false
+    asserts:
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*background_health_checks:\s*true$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_interval:\s*240$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_concurrency:\s*16$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_details:\s*false$'
+  - it: should allow overriding liveness, readiness, and startup probes
+    template: deployment.yaml
+    set:
+      livenessProbe:
+        path: /custom/livez
+        initialDelaySeconds: 5
+        periodSeconds: 15
+        timeoutSeconds: 5
+        successThreshold: 1
+        failureThreshold: 5
+      readinessProbe:
+        path: /custom/readyz
+        initialDelaySeconds: 10
+        periodSeconds: 20
+        timeoutSeconds: 6
+        successThreshold: 1
+        failureThreshold: 6
+      startupProbe:
+        path: /custom/startupz
+        initialDelaySeconds: 15
+        periodSeconds: 25
+        timeoutSeconds: 7
+        successThreshold: 1
+        failureThreshold: 40
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /custom/livez
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 5
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /custom/readyz
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 6
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.httpGet.path
+          value: /custom/startupz
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.failureThreshold
+          value: 40
+  - it: should render container resources from values
+    template: deployment.yaml
+    set:
+      resources:
+        limits:
+          cpu: 500m
+          memory: 2Gi
+        requests:
+          cpu: 250m
+          memory: 1Gi
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.cpu
+          value: 500m
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.memory
+          value: 2Gi
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.cpu
+          value: 250m
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.memory
+          value: 1Gi
+  - it: should keep default probes and empty resources unchanged
+    template: deployment.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /health/liveliness
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.failureThreshold
+          value: 3
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /health/readiness
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.failureThreshold
+          value: 3
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.httpGet.path
+          value: /health/readiness
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.failureThreshold
+          value: 30
+      - equal:
+          path: spec.template.spec.containers[0].resources
+          value: {}
+  - it: should be able to set minReadySeconds
+    template: deployment.yaml
+    set:
+      deploymentMinReadySeconds: 5
+    asserts:
+      - equal:
+          path: spec.minReadySeconds
+          value: 5
+  - it: should have minReadySeconds absent when deploymentMinReadySeconds is not set
+    template: deployment.yaml
+    asserts:
+      - notExists:
+          path: spec.minReadySeconds

deploy/charts/litellm-helm/tests/migrations-job_tests.yaml

@@ -124,4 +124,67 @@ tests:
       - notContains:
           path: spec.template.spec.containers[0].env
           content:
-            name: DATABASE_URL
+            name: DATABASE_URL
+
+  - it: should use default service account for helm hooks when serviceAccount.create is true
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: true
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: default
+
+  - it: should use migrationJob.serviceAccountName override for helm hooks when serviceAccount.create is true
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        serviceAccountName: migration-sa
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: true
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: migration-sa
+
+  - it: should use chart service account when helm hooks are disabled
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: false
+      serviceAccount:
+        create: true
+        name: my-custom-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: my-custom-sa
+
+  - it: should use pre-existing service account when helm hooks are enabled but serviceAccount.create is false
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+        hooks:
+          helm:
+            enabled: true
+      serviceAccount:
+        create: false
+        name: pre-existing-sa
+    asserts:
+      - equal:
+          path: spec.template.spec.serviceAccountName
+          value: pre-existing-sa

deploy/charts/litellm-helm/values.yaml

@@ -31,10 +31,20 @@ serviceAccount:
 # annotations for litellm deployment
 deploymentAnnotations: {}
 deploymentLabels: {}
+deploymentMinReadySeconds: 0
+
 # annotations for litellm pods
 podAnnotations: {}
 podLabels: {}
 
+# -- Deployment strategy configuration
+# Example:
+#   type: RollingUpdate
+#   rollingUpdate:
+#     maxUnavailable: 0
+#     maxSurge: 1
+strategy: {}
+
 terminationGracePeriodSeconds: 90
 topologySpreadConstraints:
   []
@@ -84,6 +94,31 @@ service:
 separateHealthApp: false
 separateHealthPort: 8081
 
+# Probe tuning for proxy container
+livenessProbe:
+  path: /health/liveliness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 3
+
+readinessProbe:
+  path: /health/readiness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 3
+
+startupProbe:
+  path: /health/readiness
+  initialDelaySeconds: 0
+  periodSeconds: 10
+  timeoutSeconds: 1
+  successThreshold: 1
+  failureThreshold: 30
+
 ingress:
   enabled: false
   className: "nginx"
@@ -156,6 +191,40 @@ autoscaling:
   targetCPUUtilizationPercentage: 80
   # targetMemoryUtilizationPercentage: 80
 
+# Autoscaling with keda is mutually exclusive with hpa
+keda:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 100
+  pollingInterval: 30
+  cooldownPeriod: 300
+  # fallback:
+  #   failureThreshold: 3
+  #   replicas: 11
+  restoreToOriginalReplicaCount: false
+  scaledObject:
+    annotations: {}
+  triggers: []
+  # - type: prometheus
+  #   metadata:
+  #     serverAddress: http://<prometheus-host>:9090
+  #     metricName: http_requests_total
+  #     threshold: '100'
+  #     query: sum(rate(http_requests_total{deployment="my-deployment"}[2m]))
+  behavior: {}
+  # scaleDown:
+  #   stabilizationWindowSeconds: 300
+  #   policies:
+  #   - type: Pods
+  #     value: 1
+  #     periodSeconds: 180
+  # scaleUp:
+  #   stabilizationWindowSeconds: 300
+  #   policies:
+  #   - type: Pods
+  #     value: 2
+  #     periodSeconds: 60
+
 # Additional volumes on the output Deployment definition.
 volumes: []
 # - name: foo
@@ -200,6 +269,14 @@ db:
   #  instance.  See the "postgresql" top level key for additional configuration.
   deployStandalone: true
 
+# Lifecycle hooks for the LiteLLM container
+# Example:
+# lifecycle:
+#   preStop:
+#     exec:
+#       command: ["/bin/sh", "-c", "sleep 10"]
+lifecycle: {}
+
 # Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored
 #  otherwise)
 postgresql:
@@ -232,6 +309,10 @@ migrationJob:
   retries: 3 # Number of retries for the Job in case of failure
   backoffLimit: 4 # Backoff limit for Job restarts
   disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
+  # Optional service account for the migration job.
+  # Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true.
+  # In that case, pre-install/pre-upgrade hooks run before normal resources, so this defaults to "default".
+  serviceAccountName: ""
   annotations: {}
   ttlSecondsAfterFinished: 120
   resources: {}
@@ -239,6 +320,7 @@ migrationJob:
   #    cpu: 100m
   #    memory: 100Mi
   extraContainers: []
+  extraInitContainers: []
 
   # Hook configuration
   hooks:

dev_config.yaml

@@ -0,0 +1,13 @@
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake-model
+      api_key: fake-key
+      api_base: https://exampleopenaiendpoint-production.up.railway.app/
+
+general_settings:
+  master_key: sk-1234
+
+litellm_settings:
+  drop_params: True
+  telemetry: False

docker-compose.hardened.yml

@@ -0,0 +1,46 @@
+services:
+  # Hardened stack: for testing the proxy under non-root, read-only, proxy-enforced constraints.
+  # Keep this file focused on hardening/QA scenarios; leave the main docker-compose.yml for default dev usage.
+  litellm:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.non_root
+      target: runtime
+      args:
+        PROXY_EXTRAS_SOURCE: "local"
+    depends_on:
+      - squid
+    user: "101:101"
+    group_add:
+      - "2345"
+    read_only: true
+    cap_drop:
+      - ALL
+    security_opt:
+      - no-new-privileges:true
+    tmpfs:
+      - /app/cache:rw,noexec,nosuid,nodev,size=128m,uid=101,gid=101,mode=1777
+      - /app/migrations:rw,noexec,nosuid,nodev,size=64m,uid=101,gid=101,mode=1777
+    volumes:
+      - ./proxy_server_config.yaml:/app/config.yaml:ro
+    environment:
+      LITELLM_NON_ROOT: "true"
+      PRISMA_BINARY_CACHE_DIR: "/app/cache/prisma-python/binaries"
+      XDG_CACHE_HOME: "/app/cache"
+      LITELLM_MIGRATION_DIR: "/app/migrations"
+      HTTP_PROXY: "http://squid:3128"
+      HTTPS_PROXY: "http://squid:3128"
+      NO_PROXY: "localhost,127.0.0.1,db"
+    command:
+      - "--port"
+      - "4000"
+      - "--config"
+      - "/app/config.yaml"
+  squid:
+    image: sameersbn/squid:3.5.27-2
+    restart: unless-stopped
+    ports:
+      - "3128:3128"
+    tmpfs:
+      - /var/spool/squid:rw,noexec,nosuid,nodev,size=64m
+      - /var/log/squid:rw,noexec,nosuid,nodev,size=16m

docker/Dockerfile.alpine

@@ -34,8 +34,8 @@ RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
 # Runtime stage
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
-# Update dependencies and clean up
-RUN apk upgrade --no-cache
+# Update dependencies and clean up, install libsndfile for audio processing
+RUN apk upgrade --no-cache && apk add --no-cache libsndfile
 
 WORKDIR /app
 
@@ -46,8 +46,9 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
-RUN chmod +x docker/entrypoint.sh
-RUN chmod +x docker/prod_entrypoint.sh
+# Convert Windows line endings to Unix for entrypoint scripts
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
+RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
 EXPOSE 4000/tcp
 

docker/Dockerfile.custom_ui

@@ -5,7 +5,41 @@ FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
 WORKDIR /app
 
 # Install Node.js and npm (adjust version as needed)
-RUN apt-get update && apt-get install -y nodejs npm
+RUN apt-get update && apt-get upgrade -y \
+        libxml2 \
+        libexpat1 \
+        openssl \
+        libssl3 \
+        git \
+        libkrb5-3 \
+        libglib2.0-0 \
+        wget \
+        libaom3 \
+        libxslt1.1 \
+        libgnutls30 \
+        libc6 && \
+    apt-get install -y nodejs npm && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
+    GLOBAL="$(npm root -g)" && \
+    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    apt-get purge -y npm
 
 # Copy the UI source into the container
 COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard
@@ -32,8 +66,9 @@ RUN rm -rf /app/litellm/proxy/_experimental/out/* && \
 WORKDIR /app
 
 # Make sure your docker/entrypoint.sh is executable
-RUN chmod +x docker/entrypoint.sh
-RUN chmod +x docker/prod_entrypoint.sh
+# Convert Windows line endings to Unix for entrypoint scripts
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
+RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
 # Expose the necessary port
 EXPOSE 4000/tcp

docker/Dockerfile.database

@@ -27,7 +27,8 @@ RUN python -m pip install build
 COPY . .
 
 # Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+# Convert Windows line endings to Unix and make executable
+RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Build the package
 RUN rm -rf dist/* && python -m build
@@ -48,7 +49,28 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # Install runtime dependencies
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
+    GLOBAL="$(npm root -g)" && \
+    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -62,21 +84,45 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
+# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
+# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
+# Patch every copy of tar, glob, and brace-expansion inside that tree.
+RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done
+
 # Install semantic_router and aurelio-sdk using script
-RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+# Convert Windows line endings to Unix and make executable
+RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
 
 # ensure pyjwt is used, not jwt
 RUN pip uninstall jwt -y
 RUN pip uninstall PyJWT -y
-RUN pip install PyJWT==2.9.0 --no-cache-dir
+RUN pip install PyJWT==2.12.0 --no-cache-dir
 
-# Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+# Build Admin UI (runtime stage)
+# Convert Windows line endings to Unix and make executable
+RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Generate prisma client
 RUN prisma generate
-RUN chmod +x docker/entrypoint.sh
-RUN chmod +x docker/prod_entrypoint.sh
+# Convert Windows line endings to Unix for entrypoint scripts
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
+RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 EXPOSE 4000/tcp
 
 RUN apk add --no-cache supervisor

docker/Dockerfile.dev

@@ -31,7 +31,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # Fix JWT dependency conflicts early
 RUN pip uninstall jwt -y || true && \
     pip uninstall PyJWT -y || true && \
-    pip install PyJWT==2.9.0 --no-cache-dir
+    pip install PyJWT==2.12.0 --no-cache-dir
 
 # Copy only necessary files for build
 COPY pyproject.toml README.md schema.prisma poetry.lock ./
@@ -40,7 +40,8 @@ COPY enterprise/ ./enterprise/
 COPY docker/ ./docker/
 
 # Build Admin UI once
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+# Convert Windows line endings to Unix and make executable
+RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Build the package
 RUN rm -rf dist/* && python -m build
@@ -55,12 +56,46 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 USER root
 
 # Install only runtime dependencies
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    libssl3 \
+RUN apt-get update && apt-get upgrade -y \
+        libxml2 \
+        libexpat1 \
+        openssl \
+        libssl3 \
+        git \
+        libkrb5-3 \
+        libglib2.0-0 \
+        wget \
+        libaom3 \
+        libxslt1.1 \
+        libgnutls30 \
+        libc6 \
+    && apt-get install -y --no-install-recommends \
+        libssl3 \
     libatomic1 \
     nodejs \
     npm \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
+    && GLOBAL="$(npm root -g)" \
+    && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+       done \
+    && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+       done \
+    && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+       done \
+    && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+       done \
+    && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+          rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+       done \
+    && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
+    && npm cache clean --force \
+    && apt-get purge -y npm
 
 WORKDIR /app
 
@@ -78,9 +113,34 @@ RUN pip install --no-cache-dir *.whl /wheels/* --no-index --find-links=/wheels/
     rm -f *.whl && \
     rm -rf /wheels
 
+# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
+# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
+# Patch every copy of tar, glob, and brace-expansion inside that tree.
+RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done
+
 # Generate prisma client and set permissions
+# Convert Windows line endings to Unix for entrypoint scripts
 RUN prisma generate && \
-    chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh
+    sed -i 's/\r$//' docker/entrypoint.sh && \
+    sed -i 's/\r$//' docker/prod_entrypoint.sh && \
+    chmod +x docker/entrypoint.sh && \
+    chmod +x docker/prod_entrypoint.sh
 
 EXPOSE 4000/tcp
 

docker/Dockerfile.health_check

@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Copy health check script and requirements
+COPY scripts/health_check/health_check_client.py /app/health_check_client.py
+COPY scripts/health_check/health_check_requirements.txt /app/requirements.txt
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Make script executable
+RUN chmod +x /app/health_check_client.py
+
+# Set entrypoint
+ENTRYPOINT ["python", "/app/health_check_client.py"]

docker/Dockerfile.non_root

@@ -1,154 +1,234 @@
 # Base images
 ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
 ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
+ARG PROXY_EXTRAS_SOURCE=published
 
 # -----------------
 # Builder Stage
 # -----------------
 FROM $LITELLM_BUILD_IMAGE AS builder
+ARG PROXY_EXTRAS_SOURCE
 WORKDIR /app
-
-# Install build dependencies including Node.js for UI build
 USER root
+
+# Install build dependencies with retry logic (includes node for UI build)
 RUN for i in 1 2 3; do \
-  apk add --no-cache \
-  python3 \
-  py3-pip \
-  clang \
-  llvm \
-  lld \
-  gcc \
-  linux-headers \
-  build-base \
-  bash \
-  nodejs \
-  npm && break || sleep 5; \
-  done \
+    apk add --no-cache \
+    python3 \
+    python3-dev \
+    py3-pip \
+    clang \
+    llvm \
+    lld \
+    gcc \
+    linux-headers \
+    build-base \
+    bash \
+    nodejs \
+    npm && break || sleep 5; \
+    done \
   && pip install --no-cache-dir --upgrade pip build
 
-# Copy project files
+# Cache Python dependencies
+COPY requirements.txt .
+RUN pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt \
+  && pip wheel --no-cache-dir --wheel-dir=/wheels/ "semantic_router==0.1.11" "aurelio-sdk==0.0.19" "PyJWT==2.12.0"
+
+# Copy source after dependency layers
 COPY . .
 
-# Set LITELLM_NON_ROOT flag for build time
+# Set non-root flag for build time consistency
 ENV LITELLM_NON_ROOT=true
 
-# Build Admin UI
-RUN mkdir -p /tmp/litellm_ui
+# Build Admin UI using the upstream command order while keeping a single RUN layer
+RUN mkdir -p /var/lib/litellm/ui && \
+    npm install -g npm@latest && npm cache clean --force && \
+    cd /app/ui/litellm-dashboard && \
+    if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
+      cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
+    fi && \
+    npm install --legacy-peer-deps && \
+    npm run build && \
+    cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
+    mkdir -p /var/lib/litellm/assets && \
+    cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
+    ( cd /var/lib/litellm/ui && \
+      for html_file in *.html; do \
+        if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
+          folder_name="${html_file%.html}" && \
+          mkdir -p "$folder_name" && \
+          mv "$html_file" "$folder_name/index.html"; \
+        fi; \
+      done && \
+      touch .litellm_ui_ready ) && \
+    cd /app/ui/litellm-dashboard && rm -rf ./out
 
-RUN npm install -g npm@latest && npm cache clean --force
-
-RUN cd /app/ui/litellm-dashboard && \
-  if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
-  cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
-  fi
-
-RUN cd /app/ui/litellm-dashboard && rm -f package-lock.json
-
-RUN cd /app/ui/litellm-dashboard && npm install --legacy-peer-deps
-
-RUN cd /app/ui/litellm-dashboard && npm run build
-
-RUN cp -r /app/ui/litellm-dashboard/out/* /tmp/litellm_ui/
-RUN mkdir -p /tmp/litellm_assets && cp /app/litellm/proxy/logo.jpg /tmp/litellm_assets/logo.jpg
-
-RUN cd /tmp/litellm_ui && \
-  for html_file in *.html; do \
-  if [ "$html_file" != "index.html" ] && [ -f "$html_file" ]; then \
-  folder_name="${html_file%.html}" && \
-  mkdir -p "$folder_name" && \
-  mv "$html_file" "$folder_name/index.html"; \
-  fi; \
-  done
-
-RUN cd /app/ui/litellm-dashboard && rm -rf ./out
-
-# Build package and wheel dependencies
+# Build litellm wheel and place it in wheels dir (replace any PyPI wheels)
 RUN rm -rf dist/* && python -m build && \
-  pip install dist/*.whl && \
-  pip wheel --no-cache-dir --wheel-dir=/wheels/ -r requirements.txt
+  rm -f /wheels/litellm-*.whl && \
+  cp dist/*.whl /wheels/
+
+# Optionally build local litellm-proxy-extras wheel
+RUN if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \
+      cd /app/litellm-proxy-extras && rm -rf dist && python -m build && \
+      cp dist/*.whl /wheels/; \
+    fi
+
+# Pre-cache Prisma binaries in the builder stage
+ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
+    PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
+    XDG_CACHE_HOME=/app/.cache \
+    PATH="/usr/lib/python3.13/site-packages/nodejs/bin:${PATH}"
+
+RUN pip install --no-cache-dir prisma==0.11.0 nodejs-wheel-binaries==24.13.1 \
+    && mkdir -p /app/.cache/npm
+
+RUN NPM_CONFIG_CACHE=/app/.cache/npm \
+    python -c "import prisma.cli.prisma as p; p.ensure_cached()"
+
+RUN prisma generate && \
+    prisma --version && \
+    prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true
 
 # -----------------
 # Runtime Stage
 # -----------------
 FROM $LITELLM_RUNTIME_IMAGE AS runtime
+ARG PROXY_EXTRAS_SOURCE
 WORKDIR /app
-
-# Install runtime dependencies
 USER root
-RUN for i in 1 2 3; do \
-  apk upgrade --no-cache && break || sleep 5; \
-  done \
-  && for i in 1 2 3; do \
-  apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
-  done
 
-# Copy only necessary artifacts from builder stage for runtime
-COPY . .
+# Install runtime dependencies with retry
+RUN for i in 1 2 3; do \
+    apk upgrade --no-cache && break || sleep 5; \
+    done \
+  && for i in 1 2 3; do \
+    apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor && break || sleep 5; \
+    done \
+  && apk upgrade --no-cache nodejs \
+  && npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
+  && GLOBAL="$(npm root -g)" \
+  && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+     done \
+  && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+     done \
+  && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+     done \
+  && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+     done \
+  && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+     done \
+  && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
+  && npm cache clean --force \
+  && { apk del --no-cache npm 2>/dev/null || true; }
+
+# Copy artifacts from builder
+COPY --from=builder /app/requirements.txt /app/requirements.txt
 COPY --from=builder /app/docker/entrypoint.sh /app/docker/prod_entrypoint.sh /app/docker/
 COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
-COPY --from=builder /app/schema.prisma /app/schema.prisma
-COPY --from=builder /app/dist/*.whl .
+COPY --from=builder /app/schema.prisma /app/
+# Copy prisma_migration.py for Helm migrations job compatibility
+COPY --from=builder /app/litellm/proxy/prisma_migration.py /app/litellm/proxy/prisma_migration.py
 COPY --from=builder /wheels/ /wheels/
-COPY --from=builder /tmp/litellm_ui /tmp/litellm_ui
-COPY --from=builder /tmp/litellm_assets /tmp/litellm_assets
+COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
+COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
+COPY --from=builder /app/.cache /app/.cache
+COPY --from=builder /app/litellm-proxy-extras /app/litellm-proxy-extras
+COPY --from=builder \
+  /usr/lib/python3.13/site-packages/nodejs* \
+  /usr/lib/python3.13/site-packages/prisma* \
+  /usr/lib/python3.13/site-packages/tomlkit* \
+  /usr/lib/python3.13/site-packages/nodeenv* \
+  /usr/lib/python3.13/site-packages/
+COPY --from=builder /usr/bin/prisma /usr/bin/prisma
 
-# Install package from wheel and dependencies
-RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ \
-  && rm -f *.whl \
-  && rm -rf /wheels
+# Final runtime environment configuration
+ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
+    PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
+    HOME=/app \
+    LITELLM_NON_ROOT=true \
+    XDG_CACHE_HOME=/app/.cache
 
-# Remove test files and keys from dependencies
-RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
-  find /usr/lib -type d -path "*/tornado/test" -delete
+# Install packages from wheels and optional extras without network
+RUN pip install --no-index --find-links=/wheels/ -r requirements.txt && \
+    pip install --no-index --find-links=/wheels/ /wheels/litellm-*-py3-none-any.whl && \
+    pip install --no-index --find-links=/wheels/ --no-deps semantic_router==0.1.11 && \
+    pip install --no-index --find-links=/wheels/ aurelio-sdk==0.0.19 && \
+    if [ "$PROXY_EXTRAS_SOURCE" = "local" ]; then \
+      if ls /wheels/litellm_proxy_extras-*.whl >/dev/null 2>&1; then \
+        pip install --no-index --find-links=/wheels/ /wheels/litellm_proxy_extras-*.whl; \
+      else \
+        echo "litellm_proxy_extras wheel not found; skipping local install"; \
+      fi; \
+    fi
 
-# Install semantic_router and aurelio-sdk using script
-RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
+# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
+# Patch every copy of tar, glob, and brace-expansion inside that tree.
+RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done
 
-# Ensure correct JWT library is used (pyjwt not jwt)
-RUN pip uninstall jwt -y && \
-  pip uninstall PyJWT -y && \
-  pip install PyJWT==2.9.0 --no-cache-dir
+# Permissions, cleanup, and Prisma prep
+# Convert Windows line endings to Unix for entrypoint scripts
+RUN sed -i 's/\r$//' docker/entrypoint.sh && \
+    sed -i 's/\r$//' docker/prod_entrypoint.sh && \
+    chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
+    mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui && \
+    chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm && \
+    pip uninstall jwt -y || true && \
+    pip uninstall PyJWT -y || true && \
+    pip install --no-index --find-links=/wheels/ PyJWT==2.12.0 --no-cache-dir && \
+    rm -rf /wheels && \
+    PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
+    chown -R nobody:nogroup $PRISMA_PATH && \
+    LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
+    [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH && \
+    LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
+    chgrp -R 0 $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
+    [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
+    chmod -R g=u $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
+    [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
+    chmod -R g+w $PRISMA_PATH /var/lib/litellm/ui /var/lib/litellm/assets && \
+    [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true && \
+    chmod -R g+rX $PRISMA_PATH && \
+    chmod -R g+rX /app/.cache && \
+    mkdir -p /tmp/.npm /nonexistent /.npm
 
-# Set Prisma cache directories
-ENV PRISMA_BINARY_CACHE_DIR=/nonexistent
-ENV NPM_CONFIG_CACHE=/.npm
-
-# Install prisma and make entrypoints executable
-RUN pip install --no-cache-dir prisma && \
-  chmod +x docker/entrypoint.sh && \
-  chmod +x docker/prod_entrypoint.sh
-
-# Create directories and set permissions for non-root user
-RUN mkdir -p /nonexistent /.npm /tmp/litellm_assets && \
-  chown -R nobody:nogroup /app /tmp/litellm_ui /tmp/litellm_assets /nonexistent /.npm && \
-  PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
-  chown -R nobody:nogroup $PRISMA_PATH && \
-  LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \
-  [ -n "$LITELLM_PKG_MIGRATIONS_PATH" ] && chown -R nobody:nogroup $LITELLM_PKG_MIGRATIONS_PATH
-
-# OpenShift compatibility
-RUN PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
-  LITELLM_PROXY_EXTRAS_PATH=$(python -c "import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))" 2>/dev/null || echo "") && \
-  chgrp -R 0 $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
-  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chgrp -R 0 $LITELLM_PROXY_EXTRAS_PATH || true && \
-  chmod -R g=u $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
-  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g=u $LITELLM_PROXY_EXTRAS_PATH || true && \
-  chmod -R g+w $PRISMA_PATH /tmp/litellm_ui /tmp/litellm_assets && \
-  [ -n "$LITELLM_PROXY_EXTRAS_PATH" ] && chmod -R g+w $LITELLM_PROXY_EXTRAS_PATH || true
-
-# Switch to non-root user
+# Switch to non-root user for runtime
 USER nobody
 
-# Set HOME for prisma generate to have a writable directory
-ENV HOME=/app
-
-# Set LITELLM_NON_ROOT flag for runtime
-ENV LITELLM_NON_ROOT=true
-
+# Generate Prisma client as nobody user to ensure correct file ownership
 RUN prisma generate
 
+# Prisma runtime knobs for offline containers
+ENV PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
+    PRISMA_HIDE_UPDATE_MESSAGE=1 \
+    PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
+    NPM_CONFIG_CACHE=/app/.cache/npm \
+    NPM_CONFIG_PREFER_OFFLINE=true \
+    PRISMA_OFFLINE_MODE=true
+
 EXPOSE 4000/tcp
-
 ENTRYPOINT ["/app/docker/prod_entrypoint.sh"]
-
-CMD ["--port", "4000"]
+CMD ["--port", "4000"]

docker/README.md

@@ -59,6 +59,33 @@ To stop the running containers, use the following command:
 docker compose down
 ```
 
+## Hardened / Offline Testing
+
+To ensure changes are safe for non-root, read-only root filesystems and restricted egress, always validate with the hardened compose file:
+
+```bash
+docker compose -f docker-compose.yml -f docker-compose.hardened.yml build --no-cache
+docker compose -f docker-compose.yml -f docker-compose.hardened.yml up -d
+```
+
+This setup:
+- Builds from `docker/Dockerfile.non_root` with Prisma engines and Node toolchain baked into the image.
+- Runs the proxy as a non-root user with a read-only rootfs and only writable tmpfs mounts:
+  - `/app/cache` (Prisma/NPM cache; backing `PRISMA_BINARY_CACHE_DIR`, `NPM_CONFIG_CACHE`, `XDG_CACHE_HOME`)
+  - `/app/migrations` (Prisma migration workspace; backing `LITELLM_MIGRATION_DIR`)
+- Pre-builds and serves the admin UI from read-only paths:
+  - `/var/lib/litellm/ui` (pre-restructured Next.js UI with `.litellm_ui_ready` marker)
+  - `/var/lib/litellm/assets` (UI logos and assets)
+- Routes all outbound traffic through a local Squid proxy that denies egress, so Prisma migrations must use the cached CLI and engines.
+
+You should also verify offline Prisma behaviour with:
+
+```bash
+docker run --rm --network none --entrypoint prisma ghcr.io/berriai/litellm:main-stable --version
+```
+
+This command should succeed (showing engine versions) even with `--network none`, confirming that Prisma binaries are available without network access.
+
 ## Troubleshooting
 
 -   **`build_admin_ui.sh: not found`**: This error can occur if the Docker build context is not set correctly. Ensure that you are running the `docker-compose` command from the root of the project.

docker/prod_entrypoint.sh

@@ -2,6 +2,7 @@
 
 if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
     export LITELLM_ARGS="$@"
+    export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
     exec supervisord -c /etc/supervisord.conf
 fi
 

docker/supervisord.conf

@@ -1,6 +1,8 @@
 [supervisord]
 nodaemon=true
 loglevel=info
+logfile=/tmp/supervisord.log
+pidfile=/tmp/supervisord.pid
 
 [group:litellm]
 programs=main,health
@@ -14,6 +16,7 @@ priority=1
 exitcodes=0
 stopasgroup=true
 killasgroup=true
+stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
 stdout_logfile=/dev/stdout
 stderr_logfile=/dev/stderr
 stdout_logfile_maxbytes = 0
@@ -29,6 +32,7 @@ priority=2
 exitcodes=0
 stopasgroup=true
 killasgroup=true
+stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
 stdout_logfile=/dev/stdout
 stderr_logfile=/dev/stderr
 stdout_logfile_maxbytes = 0

docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md

@@ -6,7 +6,7 @@ authors:
   - name: Sameer Kankute
     title: SWE @ LiteLLM (LLM Translation)
     url: https://www.linkedin.com/in/sameer-kankute/
-    image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII
+    image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg
   - name: Krrish Dholakia
     title: "CEO, LiteLLM"
     url: https://www.linkedin.com/in/krish-d/
@@ -15,6 +15,7 @@ authors:
     title: "CTO, LiteLLM"
     url: https://www.linkedin.com/in/reffajnaahsi/
     image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
+description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter."
 tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features]
 hide_table_of_contents: false
 ---