fix: resolve merge conflicts with main

- Update load_local_model_cost_map to use project root fallback for dev
- Keep main's validation, aliases, and source info tracking
- Remove backup JSON (purpose of this PR)

.circleci/requirements.txt

@@ -8,12 +8,14 @@ redis==5.2.1
 redisvl==0.4.1
 anthropic
 orjson==3.10.12 # fast /embedding responses
-pydantic==2.10.2
+pydantic==2.11.0
 google-cloud-aiplatform==1.43.0
 google-cloud-iam==2.19.1
 fastapi-sso==0.16.0 
 uvloop==0.21.0
-mcp==1.10.1    # for MCP server  
+mcp==1.25.0    # for MCP server  
 semantic_router==0.1.10 # for auto-routing with litellm
 fastuuid==0.12.0
-responses==0.25.7  # for proxy client tests
+responses==0.25.7  # for proxy client tests
+pytest-retry==1.6.3  # for automatic test retries
+litellm-proxy-extras  # for prisma migrations

.claude/settings.json

@@ -0,0 +1,36 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git show:*)",
+      "Bash(git worktree add:*)",
+      "Read(//Users/krrishdholakia/Documents/litellm/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/**)",
+      "Bash(python:*)",
+      "Bash(python -c \"\nimport sys; sys.path.insert\\(0, ''.''\\)\nfrom litellm.proxy.guardrails.guardrail_hooks.claude_code.guardrail import ClaudeCodeGuardrail, HOSTED_TOOL_PREFIXES\nprint\\(''HOSTED_TOOL_PREFIXES:'', HOSTED_TOOL_PREFIXES\\)\nprint\\(''ClaudeCodeGuardrail imported OK''\\)\n\")",
+      "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy/**)",
+      "Read(//Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/**)",
+      "Bash(poetry run pytest:*)",
+      "Bash(git add:*)",
+      "Bash(git commit:*)",
+      "Bash(poetry run python:*)",
+      "Bash(poetry run pip:*)",
+      "Bash(git reset:*)",
+      "Bash(git cherry-pick:*)",
+      "Bash(git checkout:*)",
+      "Read(//Users/krrishdholakia/Documents/litellm/litellm/proxy/guardrails/guardrail_hooks/**)",
+      "Read(//Users/krrishdholakia/Documents/**)",
+      "Bash(git -C /Users/krrishdholakia/Documents/litellm-mcp-user-permissions worktree list)",
+      "Bash(ls:*)"
+    ],
+    "additionalDirectories": [
+      "/Users/krrishdholakia/Documents/litellm-mcp-group-plan/plan",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/proxy/guardrails/guardrail_hooks/claude_code",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails/litellm/types",
+      "/Users/krrishdholakia/Documents/litellm-claude-code-guardrails",
+      "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/litellm/proxy",
+      "/Users/krrishdholakia/Documents/litellm-mcp-jwt-groups/tests/test_litellm/proxy/auth"
+    ]
+  }
+}

.dockerignore

@@ -48,7 +48,7 @@ dist/
 build/
 *.egg-info/
 .DS_Store
-node_modules/
+**/node_modules
 *.log
 .env
 .env.local

.gitguardian.yaml

@@ -0,0 +1,111 @@
+version: 2
+
+secret:
+  # Exclude files and paths by globbing
+  ignored_paths:
+    - "**/*.whl"
+    - "**/*.pyc"
+    - "**/__pycache__/**"
+    - "**/node_modules/**"
+    - "**/dist/**"
+    - "**/build/**"
+    - "**/.git/**"
+    - "**/venv/**"
+    - "**/.venv/**"
+    
+    # Large data/metadata files that don't need scanning
+    - "**/model_prices_and_context_window*.json"
+    - "**/*_metadata/*.txt"
+    - "**/tokenizers/*.json"
+    - "**/tokenizers/*"
+    - "miniconda.sh"
+    
+    # Build outputs and static assets
+    - "litellm/proxy/_experimental/out/**"
+    - "ui/litellm-dashboard/public/**"
+    - "**/swagger/*.js"
+    - "**/*.woff"
+    - "**/*.woff2"
+    - "**/*.avif"
+    - "**/*.webp"
+    
+    # Test data files
+    - "**/tests/**/data_map.txt"
+    - "tests/**/*.txt"
+    
+    # Documentation and other non-code files
+    - "docs/**"
+    - "**/*.md"
+    - "**/*.lock"
+    - "poetry.lock"
+    - "package-lock.json"
+
+  # Ignore security incidents with the SHA256 of the occurrence (false positives)
+  ignored_matches:
+    # === Current detected false positives (SHA-based) ===
+    
+    # gcs_pub_sub_body - folder name, not a password
+    - name: GCS pub/sub test folder name
+      match: 75f377c456eede69e5f6e47399ccee6016a2a93cc5dd11db09cc5b1359ae569a
+    
+    # os.environ/APORIA_API_KEY_1 - environment variable reference
+    - name: Environment variable reference APORIA_API_KEY_1
+      match: e2ddeb8b88eca97a402559a2be2117764e11c074d86159ef9ad2375dea188094
+    
+    # os.environ/APORIA_API_KEY_2 - environment variable reference
+    - name: Environment variable reference APORIA_API_KEY_2
+      match: 09aa39a29e050b86603aa55138af1ff08fb86a4582aa965c1bd0672e1575e052
+    
+    # oidc/circleci_v2/ - test authentication path, not a secret
+    - name: OIDC CircleCI test path
+      match: feb3475e1f89a65b7b7815ac4ec597e18a9ec1847742ad445c36ca617b536e15
+    
+    # text-davinci-003 - OpenAI model identifier, not a secret
+    - name: OpenAI model identifier text-davinci-003
+      match: c489000cf6c7600cee0eefb80ad0965f82921cfb47ece880930eb7e7635cf1f1
+    
+    # Base64 Basic Auth in test_pass_through_endpoints.py - test fixture, not a real secret
+    - name: Test Base64 Basic Auth header in pass_through_endpoints test
+      match: 61bac0491f395040617df7ef6d06029eac4d92a4457ac784978db80d97be1ae0
+    
+    # PostgreSQL password "postgres" in CI configs - standard test database password
+    - name: Test PostgreSQL password in CI configurations
+      match: 6e0d657eb1f0fbc40cf0b8f3c3873ef627cc9cb7c4108d1c07d979c04bc8a4bb
+    
+    # Bearer token in locustfile.py - test/example API key for load testing
+    - name: Test Bearer token in locustfile load test
+      match: 2a0abc2b0c3c1760a51ffcdf8d6b1d384cef69af740504b1cfa82dd70cdc7ff9
+    
+    # Inkeep API key in docusaurus.config.js - public documentation site key
+    - name: Inkeep API key in documentation config
+      match: c366657791bfb5fc69045ec11d49452f09a0aebbc8648f94e2469b4025e29a75
+    
+    # Langfuse credentials in test_completion.py - test credentials for integration test
+    - name: Langfuse test credentials in test_completion
+      match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
+    
+    # Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
+    - name: Test password in e2e test fixtures
+      match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
+    
+    # === Preventive patterns for test keys (pattern-based) ===
+    
+    # Test API keys (124 instances across 45 files)
+    - name: Test API keys with sk-test prefix
+      match: sk-test-
+    
+    # Mock API keys
+    - name: Mock API keys with sk-mock prefix
+      match: sk-mock-
+    
+    # Fake API keys
+    - name: Fake API keys with sk-fake prefix
+      match: sk-fake-
+    
+    # Generic test API key patterns
+    - name: Test API key patterns
+      match: test-api-key
+
+    - name: Short fake sk keys (1–9 digits only)
+      match: \bsk-\d{1,9}\b
+

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -7,6 +7,16 @@ body:
     attributes:
       value: |
         Thanks for taking the time to fill out this bug report!
+        
+        **💡 Tip:** See our [Troubleshooting Guide](https://docs.litellm.ai/docs/troubleshoot) for what information to include.
+  - type: checkboxes
+    id: duplicate-check
+    attributes:
+      label: Check for existing issues
+      description: Please search to see if an issue already exists for the bug you encountered.
+      options:
+        - label: I have searched the existing issues and checked that my issue is not a duplicate.
+          required: true
   - type: textarea
     id: what-happened
     attributes:
@@ -16,6 +26,21 @@ body:
       value: "A bug happened!"
     validations:
       required: true
+  - type: textarea
+    id: steps-to-reproduce
+    attributes:
+      label: Steps to Reproduce
+      description: Please provide detailed steps to reproduce this bug(A curl/python code to reproduce the bug)
+      placeholder: |
+        1. config.yaml file/ .env file/ etc.
+        2. Run the following code...
+        3. Observe the error...
+      value: |
+        1. 
+        2. 
+        3. 
+    validations:
+      required: true
   - type: textarea
     id: logs
     attributes:
@@ -23,13 +48,16 @@ body:
       description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
       render: shell
   - type: dropdown
-    id: ml-ops-team
+    id: component
     attributes:
-      label: Are you a ML Ops Team?
-      description: This helps us prioritize your requests correctly 
+      label: What part of LiteLLM is this about?
       options:
-        - "No"
-        - "Yes"
+        - ''
+        - "SDK (litellm Python package)"
+        - "Proxy"
+        - "UI Dashboard"
+        - "Docs"
+        - "Other"
     validations:
       required: true
   - type: input

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,7 @@
 blank_issues_enabled: true
 contact_links:
   - name: Schedule Demo
-    url: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat
+    url: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions
     about: Speak directly with Krrish and Ishaan, the founders, to discuss issues, share feedback, or explore improvements for LiteLLM
   - name: Discord
     url: https://discord.com/invite/wuPM9dRgDw

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -7,6 +7,14 @@ body:
     attributes:
       value: |
         Thanks for making LiteLLM better! 
+  - type: checkboxes
+    id: duplicate-check
+    attributes:
+      label: Check for existing issues
+      description: Please search to see if an issue already exists for the feature you are requesting.
+      options:
+        - label: I have searched the existing issues and checked that my issue is not a duplicate.
+          required: true
   - type: textarea
     id: the-feature
     attributes:
@@ -22,6 +30,19 @@ body:
       description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too.
     validations:
       required: true
+  - type: dropdown
+    id: component
+    attributes:
+      label: What part of LiteLLM is this about?
+      options:
+        - ''
+        - "SDK (litellm Python package)"
+        - "Proxy"
+        - "UI Dashboard"
+        - "Docs"
+        - "Other"
+    validations:
+      required: true
   - type: dropdown
     id: hiring-interest
     attributes:

.github/actions/helm-oci-chart-releaser/action.yml

@@ -40,38 +40,33 @@ outputs:
 runs:
   using: composite
   steps:
+    - name: Helm | Setup
+      uses: azure/setup-helm@v4
+      with:
+        version: v3.20.0
+
     - name: Helm | Login
       shell: bash
       run: echo ${{ inputs.registry_password }} | helm registry login -u ${{ inputs.registry_username }} --password-stdin ${{ inputs.registry }}
-      env:
-        HELM_EXPERIMENTAL_OCI: '1'
-    
+
     - name: Helm | Dependency
       if: inputs.update_dependencies == 'true'
       shell: bash
       run: helm dependency update ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }}
-      env:
-        HELM_EXPERIMENTAL_OCI: '1'
 
     - name: Helm | Package
       shell: bash
       run: helm package ${{ inputs.path == null && format('{0}/{1}', 'charts', inputs.name) || inputs.path }} --version ${{ inputs.tag }} --app-version ${{ inputs.app_version }}
-      env:
-        HELM_EXPERIMENTAL_OCI: '1'
 
     - name: Helm | Push
       shell: bash
       run: helm push ${{ inputs.name }}-${{ inputs.tag }}.tgz oci://${{ inputs.registry }}/${{ inputs.repository }}
-      env:
-        HELM_EXPERIMENTAL_OCI: '1'
 
     - name: Helm | Logout
       shell: bash
       run: helm registry logout ${{ inputs.registry }}
-      env:
-        HELM_EXPERIMENTAL_OCI: '1'
 
     - name: Helm | Output
       id: output
       shell: bash
-      run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT
+      run: echo "image=${{ inputs.registry }}/${{ inputs.repository }}/${{ inputs.name }}:${{ inputs.tag }}" >> $GITHUB_OUTPUT

.github/codeql/codeql-config.yml

@@ -0,0 +1,22 @@
+name: "LiteLLM CodeQL config"
+
+# Use security-extended suite instead of security-and-quality to avoid
+# result sets > 2 GiB on this codebase that cause fatal OOM failures.
+queries:
+  - uses: security-extended
+
+# These two queries are security queries included in security-extended that
+# individually produce result sets > 2 GiB on this codebase, causing fatal
+# OOM failures. Exclude them as a safety net until CI confirms they no longer
+# OOM; drop these exclusions in a follow-up once verified.
+query-filters:
+  - exclude:
+      id: py/clear-text-logging-sensitive-data  # CWE-312 — > 2 GiB result set
+  - exclude:
+      id: py/polynomial-redos                   # CWE-730 — > 2 GiB result set
+
+paths-ignore:
+  - tests
+  - docs
+  - "**/*.md"
+  - litellm/proxy/_experimental/out

.github/observatory/litellm_config.yaml

@@ -0,0 +1,19 @@
+# LiteLLM Observatory Test Configuration
+# This config is used by CI to spin up a temporary LiteLLM instance
+# for running observatory tests against RC/stable releases.
+#
+# Add model definitions for the providers you want to test.
+# Provider API keys are injected via environment variables in CI.
+
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: azure/gpt-4o
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE
+
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: azure/gpt-4o-mini
+      api_key: os.environ/AZURE_API_KEY
+      api_base: os.environ/AZURE_API_BASE

.github/pull_request_template.md

@@ -1,7 +1,3 @@
-## Title
-
-<!-- e.g. "Implement user authentication feature" -->
-
 ## Relevant issues
 
 <!-- e.g. "Fixes #000" -->
@@ -10,11 +6,27 @@
 
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
-- [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
-- [ ] I have added a screenshot of my new test passing locally 
+- [ ] I have Added testing in the [`tests/test_litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/test_litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code)
 - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem
+- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
 
+## CI (LiteLLM team)
+
+> **CI status guideline:**
+>
+> - 50-55 passing tests: main is stable with minor issues.
+> - 45-49 passing tests: acceptable but needs attention
+> - <= 40 passing tests: unstable; be careful with your merges and assess the risk.
+
+- [ ] **Branch creation CI run**  
+       Link:
+
+- [ ] **CI run for the last commit**  
+       Link:
+
+- [ ] **Merge / cherry-pick CI run**  
+       Links:
 
 ## Type
 
@@ -29,5 +41,3 @@
 ✅ Test
 
 ## Changes
-
-

.github/scripts/close_duplicate_issues.py

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""
+Detect and close duplicate GitHub issues using title similarity.
+
+Modes:
+  --scan           Compare all open issues against each other (batch)
+  --issue-number N Check a single issue against older open issues
+
+Requires the `gh` CLI to be authenticated.
+"""
+
+import argparse
+import difflib
+import json
+import re
+import subprocess
+import sys
+
+
+def normalize_title(title: str) -> str:
+    """Strip common prefixes, lowercase, and collapse whitespace."""
+    title = re.sub(
+        r"^\[?(bug|feature request|enhancement|question|docs)[:\]]?\s*",
+        "",
+        title,
+        flags=re.IGNORECASE,
+    )
+    return " ".join(title.lower().split())
+
+
+def gh(*args: str) -> str:
+    """Run a gh CLI command and return stdout."""
+    result = subprocess.run(
+        ["gh", *args],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def fetch_open_issues(repo: str | None) -> list[dict]:
+    """Fetch all open issues (excluding PRs) via gh api --paginate."""
+    if repo:
+        endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    else:
+        endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
+    cmd = ["api", "--paginate", endpoint]
+
+    raw = gh(*cmd)
+    # gh --paginate concatenates JSON arrays, so we may get multiple arrays
+    issues = []
+    for line in raw.strip().splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        parsed = json.loads(line)
+        if isinstance(parsed, list):
+            issues.extend(parsed)
+        else:
+            issues.append(parsed)
+
+    # Filter out pull requests (they also appear in the issues endpoint)
+    return [i for i in issues if "pull_request" not in i]
+
+
+def close_as_duplicate(
+    issue_number: int, duplicate_of: int, repo: str | None, dry_run: bool
+) -> None:
+    """Close an issue as duplicate of another, adding a comment and label."""
+    repo_args = ["--repo", repo] if repo else []
+
+    if dry_run:
+        print(f"  [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
+        return
+
+    # Add comment
+    comment_body = (
+        f"Closing as duplicate of #{duplicate_of}.\n\n"
+        "If you believe this is not a duplicate, please reopen and add context "
+        "explaining how this differs."
+    )
+    gh("issue", "comment", str(issue_number), "--body", comment_body, *repo_args)
+
+    # Add label
+    gh("issue", "edit", str(issue_number), "--add-label", "duplicate", *repo_args)
+
+    # Close with not_planned reason
+    gh(
+        "api",
+        f"repos/{repo or '{owner}/{repo}'}/issues/{issue_number}",
+        "-X",
+        "PATCH",
+        "-f",
+        "state=closed",
+        "-f",
+        "state_reason=not_planned",
+    )
+
+    print(f"  Closed #{issue_number} as duplicate of #{duplicate_of}")
+
+
+def find_duplicate(
+    issue: dict, candidates: list[dict], threshold: float
+) -> dict | None:
+    """Return the first candidate whose normalized title is above threshold."""
+    norm = normalize_title(issue["title"])
+    for candidate in candidates:
+        if candidate["number"] == issue["number"]:
+            continue
+        cand_norm = normalize_title(candidate["title"])
+        ratio = difflib.SequenceMatcher(None, norm, cand_norm).ratio()
+        if ratio >= threshold:
+            return candidate
+    return None
+
+
+def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
+    """Compare every issue against all older issues. Returns count of duplicates found."""
+    # Sort oldest first
+    issues.sort(key=lambda i: i["number"])
+    closed_count = 0
+
+    for idx, issue in enumerate(issues):
+        older = issues[:idx]
+        if not older:
+            continue
+        dup = find_duplicate(issue, older, threshold)
+        if dup:
+            ratio = difflib.SequenceMatcher(
+                None,
+                normalize_title(issue["title"]),
+                normalize_title(dup["title"]),
+            ).ratio()
+            print(
+                f"#{issue['number']}: \"{issue['title']}\"\n"
+                f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+                f"({ratio:.0%} similar)"
+            )
+            close_as_duplicate(issue["number"], dup["number"], repo, dry_run)
+            closed_count += 1
+
+    return closed_count
+
+
+def check_single(
+    issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
+) -> bool:
+    """Check a single issue against all older open issues. Returns True if duplicate found."""
+    target = None
+    for i in issues:
+        if i["number"] == issue_number:
+            target = i
+            break
+
+    if target is None:
+        print(f"Issue #{issue_number} not found among open issues.")
+        return False
+
+    older = [i for i in issues if i["number"] < issue_number]
+    dup = find_duplicate(target, older, threshold)
+    if dup:
+        ratio = difflib.SequenceMatcher(
+            None,
+            normalize_title(target["title"]),
+            normalize_title(dup["title"]),
+        ).ratio()
+        print(
+            f"#{target['number']}: \"{target['title']}\"\n"
+            f"  -> duplicate of #{dup['number']}: \"{dup['title']}\" "
+            f"({ratio:.0%} similar)"
+        )
+        close_as_duplicate(issue_number, dup["number"], repo, dry_run)
+        return True
+
+    print(f"#{issue_number}: no duplicate found above threshold {threshold}")
+    return False
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--scan", action="store_true", help="Scan all open issues")
+    mode.add_argument("--issue-number", type=int, help="Check a single issue number")
+    parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
+    parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
+    parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
+    args = parser.parse_args()
+
+    dry_run = not args.close
+
+    if dry_run:
+        print("=== DRY RUN MODE (pass --close to actually close issues) ===\n")
+
+    print("Fetching open issues...")
+    issues = fetch_open_issues(args.repo)
+    print(f"Found {len(issues)} open issues.\n")
+
+    if args.scan:
+        count = scan_all(issues, args.threshold, args.repo, dry_run)
+        print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
+    else:
+        found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
+        sys.exit(0 if found else 0)  # Always exit 0; finding no dup is not an error
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/auto_update_price_and_context_window.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   auto_update_price_and_context_window:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

.github/workflows/check_duplicate_issues.yml

@@ -0,0 +1,52 @@
+name: Check Duplicate Issues
+
+on:
+  issues:
+    types: [opened, edited]
+
+jobs:
+  check-duplicate:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Check for potential duplicates
+        uses: wow-actions/potential-duplicates@v1
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          label: potential-duplicate
+          threshold: 0.6
+          reaction: eyes
+          comment: |
+            **⚠️ Potential duplicate detected**
+
+            This issue appears similar to existing issue(s):
+            {{#issues}}
+            - [#{{number}}]({{html_url}}) - {{title}} ({{accuracy}}% similar)
+            {{/issues}}
+
+            Please review the linked issue(s) to see if they address your concern. If this is not a duplicate, please provide additional context to help us understand the difference.
+
+      - name: Checkout close script
+        if: github.event.action == 'opened'
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        if: github.event.action == 'opened'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Auto-close if high-confidence duplicate
+        if: github.event.action == 'opened'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 .github/scripts/close_duplicate_issues.py \
+            --issue-number ${{ github.event.issue.number }} \
+            --repo ${{ github.repository }} \
+            --threshold 0.85 \
+            --close

.github/workflows/codeql.yml

@@ -0,0 +1,52 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    # Run weekly on Sundays at 04:00 UTC
+    - cron: "0 4 * * 0"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      security-events: write
+      packages: read
+      actions: read
+      contents: read
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: actions
+            build-mode: none
+          - language: javascript-typescript
+            build-mode: none
+          - language: python
+            build-mode: none
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+          build-mode: ${{ matrix.build-mode }}
+          config-file: ./.github/codeql/codeql-config.yml
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{ matrix.language }}"

.github/workflows/create_daily_staging_branch.yml

@@ -0,0 +1,79 @@
+name: Create Daily Staging Branch
+
+on:
+  schedule:
+    - cron: '0 0,12 * * *'  # Runs every 12 hours at midnight and noon UTC
+  workflow_dispatch:  # Allow manual trigger
+
+jobs:
+  create-staging-branch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Create daily staging branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Configure Git user
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Generate branch name with MM_DD_YYYY format
+          BRANCH_NAME="litellm_oss_staging_$(date +'%m_%d_%Y')"
+          echo "Creating branch: $BRANCH_NAME"
+
+          # Fetch all branches
+          git fetch --all
+
+          # Check if the branch already exists
+          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+            echo "Branch $BRANCH_NAME already exists. Skipping creation."
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            # Create the new branch from main
+            git checkout -b $BRANCH_NAME origin/main
+            # Push the new branch
+            git push origin $BRANCH_NAME
+            echo "Successfully created and pushed branch: $BRANCH_NAME"
+          fi
+
+  create-internal-dev-branch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Create internal dev branch
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Configure Git user
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Generate branch name with MM_DD_YYYY format
+          BRANCH_NAME="litellm_internal_dev_$(date +'%m_%d_%Y')"
+          echo "Creating branch: $BRANCH_NAME"
+
+          # Fetch all branches
+          git fetch --all
+
+          # Check if the branch already exists
+          if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then
+            echo "Branch $BRANCH_NAME already exists. Skipping creation."
+          else
+            echo "Creating new branch: $BRANCH_NAME"
+            # Create the new branch from main
+            git checkout -b $BRANCH_NAME origin/main
+            # Push the new branch
+            git push origin $BRANCH_NAME
+            echo "Successfully created and pushed branch: $BRANCH_NAME"
+          fi

.github/workflows/ghcr_deploy.yml

@@ -5,6 +5,7 @@ on:
     inputs:
       tag:
         description: "The tag version you want to build"
+        required: true
       release_type:
         description: "The release type you want to build. Can be 'latest', 'stable', 'dev', 'rc'"
         type: string
@@ -298,6 +299,15 @@ jobs:
             ${{ github.event.inputs.release_type == 'stable' && format('{0}/berriai/litellm-spend_logs:main-stable', env.REGISTRY) || '' }}
           platforms: local,linux/amd64,linux/arm64,linux/arm64/v8
 
+  run-observatory-tests:
+    if: github.event.inputs.release_type == 'rc' || github.event.inputs.release_type == 'stable'
+    needs: [docker-hub-deploy]
+    uses: ./.github/workflows/run_observatory_tests.yml
+    with:
+      tag: ${{ github.event.inputs.tag }}
+      commit_hash: ${{ github.event.inputs.commit_hash }}
+    secrets: inherit
+
   build-and-push-helm-chart:
     if: github.event.inputs.release_type  != 'dev'
     needs: [docker-hub-deploy, build-and-push-image, build-and-push-image-database]
@@ -319,44 +329,37 @@ jobs:
         run: |
           echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
     
-      - name: Get LiteLLM Latest Tag
-        id: current_app_tag
+      # Sync Helm chart version with LiteLLM release version (1-1 versioning)
+      # This allows users to easily map Helm chart versions to LiteLLM versions
+      # See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/
+      - name: Calculate chart and app versions
+        id: chart_version
         shell: bash
         run: |
-          LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0)
-          if [ -z "${LATEST_TAG}" ]; then
-            echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT
-          else
-            echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT
+          INPUT_TAG="${{ github.event.inputs.tag }}"
+          RELEASE_TYPE="${{ github.event.inputs.release_type }}"
+
+          # Chart version = LiteLLM version without 'v' prefix (Helm semver convention)
+          # v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1
+          CHART_VERSION="${INPUT_TAG#v}"
+
+          # Add suffix for 'latest' releases (rc already has suffix in tag)
+          if [ "$RELEASE_TYPE" = "latest" ]; then
+            CHART_VERSION="${CHART_VERSION}-latest"
           fi
 
-      - name: Get last published chart version
-        id: current_version
-        shell: bash
-        run: |
-          CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
-          if [ -z "${CHART_LIST}" ]; then
-            echo "current-version=0.1.0" | tee -a $GITHUB_OUTPUT
-          else
-            printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
-          fi
-        env:
-          HELM_EXPERIMENTAL_OCI: '1'
+          # App version = Docker tag (keeps 'v' prefix to match Docker image tags)
+          APP_VERSION="${INPUT_TAG}"
 
-      # Automatically update the helm chart version one "patch" level
-      - name: Bump release version
-        id: bump_version
-        uses: christian-draeger/increment-semantic-version@1.1.0
-        with:
-          current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
-          version-fragment: 'bug'
+          echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
+          echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
 
       - uses: ./.github/actions/helm-oci-chart-releaser
         with:
           name: ${{ env.CHART_NAME }}
           repository: ${{ env.REPO_OWNER }}
-          tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
-          app_version: ${{ steps.current_app_tag.outputs.latest_tag }}
+          tag: ${{ steps.chart_version.outputs.version }}
+          app_version: ${{ steps.chart_version.outputs.app_version }}
           path: deploy/charts/${{ env.CHART_NAME }}
           registry: ${{ env.REGISTRY }}
           registry_username: ${{ github.actor }}

.github/workflows/ghcr_helm_deploy.yml

@@ -1,10 +1,12 @@
-# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
+# Standalone workflow to publish LiteLLM Helm Chart
+# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release
 name: Build, Publish LiteLLM Helm Chart. New Release
 on:
   workflow_dispatch:
     inputs:
-      chartVersion:
-        description: "Update the helm chart's version to this"
+      tag:
+        description: "LiteLLM version tag (e.g., v1.81.0)"
+        required: true
 
 # Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
@@ -31,24 +33,22 @@ jobs:
         run: |
           echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
 
-      - name: Get LiteLLM Latest Tag
-        id: current_app_tag
-        uses: WyriHaximus/github-action-get-previous-tag@v1.3.0
-
-      - name: Get last published chart version
-        id: current_version
+      # Sync Helm chart version with LiteLLM release version (1-1 versioning)
+      - name: Calculate chart and app versions
+        id: chart_version
         shell: bash
-        run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
-        env:
-          HELM_EXPERIMENTAL_OCI: '1'
+        run: |
+          INPUT_TAG="${{ github.event.inputs.tag }}"
 
-      # Automatically update the helm chart version one "patch" level
-      - name: Bump release version
-        id: bump_version
-        uses: christian-draeger/increment-semantic-version@1.1.0
-        with:
-          current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
-          version-fragment: 'bug'
+          # Chart version = LiteLLM version without 'v' prefix
+          # v1.81.0 -> 1.81.0
+          CHART_VERSION="${INPUT_TAG#v}"
+
+          # App version = Docker tag (keeps 'v' prefix)
+          APP_VERSION="${INPUT_TAG}"
+
+          echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
+          echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
 
       - name: Lint helm chart
         run: helm lint deploy/charts/litellm-helm
@@ -57,8 +57,8 @@ jobs:
         with:
           name: litellm-helm
           repository: ${{ env.REPO_OWNER }}
-          tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
-          app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
+          tag: ${{ steps.chart_version.outputs.version }}
+          app_version: ${{ steps.chart_version.outputs.app_version }}
           path: deploy/charts/litellm-helm
           registry: ${{ env.REGISTRY }}
           registry_username: ${{ github.actor }}

.github/workflows/interpret_load_test.py

@@ -123,7 +123,7 @@ if __name__ == "__main__":
         + docker_run_command
         + "\n\n"
         + "### Don't want to maintain your internal proxy? get in touch 🎉"
-        + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat"
+        + "\nHosted Proxy Alpha: https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions"
         + "\n\n"
         + "## Load Test LiteLLM Proxy Results"
         + "\n\n"

.github/workflows/issue-keyword-labeler.yml

@@ -19,7 +19,7 @@ jobs:
         id: scan
         env:
           PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }}
-          KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic
+          KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic,gemini,cohere,mistral,groq,ollama,deepseek
         run: python3 .github/scripts/scan_keywords.py
 
       - name: Ensure label exists

.github/workflows/label-component.yml

@@ -0,0 +1,116 @@
+name: Label Component Issues
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-component-label:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+      - name: Add component labels
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const body = context.payload.issue.body;
+            if (!body) return;
+
+            // Define component mappings with regex patterns that handle flexible whitespace
+            const components = [
+              {
+                pattern: /What part of LiteLLM is this about\?\s*SDK \(litellm Python package\)/,
+                label: 'sdk',
+                color: '0E7C86',
+                description: 'Issues related to the litellm Python SDK'
+              },
+              {
+                pattern: /What part of LiteLLM is this about\?\s*Proxy/,
+                label: 'proxy',
+                color: '5319E7',
+                description: 'Issues related to the LiteLLM Proxy'
+              },
+              {
+                pattern: /What part of LiteLLM is this about\?\s*UI Dashboard/,
+                label: 'ui-dashboard',
+                color: 'D876E3',
+                description: 'Issues related to the LiteLLM UI Dashboard'
+              },
+              {
+                pattern: /What part of LiteLLM is this about\?\s*Docs/,
+                label: 'docs',
+                color: 'FBCA04',
+                description: 'Issues related to LiteLLM documentation'
+              }
+            ];
+
+            // Find matching component
+            for (const component of components) {
+              if (component.pattern.test(body)) {
+                // Ensure label exists
+                try {
+                  await github.rest.issues.getLabel({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    name: component.label
+                  });
+                } catch (error) {
+                  if (error.status === 404) {
+                    await github.rest.issues.createLabel({
+                      owner: context.repo.owner,
+                      repo: context.repo.repo,
+                      name: component.label,
+                      color: component.color,
+                      description: component.description
+                    });
+                  }
+                }
+
+                // Add label to issue
+                await github.rest.issues.addLabels({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: context.issue.number,
+                  labels: [component.label]
+                });
+
+                break;
+              }
+            }
+
+            // Check for 'claude code' keyword (can be applied alongside component labels)
+            if (/claude code/i.test(body)) {
+              const claudeLabel = {
+                name: 'claude code',
+                color: '7c3aed',
+                description: 'Issues related to Claude Code usage'
+              };
+
+              try {
+                await github.rest.issues.getLabel({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  name: claudeLabel.name
+                });
+              } catch (error) {
+                if (error.status === 404) {
+                  await github.rest.issues.createLabel({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    name: claudeLabel.name,
+                    color: claudeLabel.color,
+                    description: claudeLabel.description
+                  });
+                }
+              }
+
+              await github.rest.issues.addLabels({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                labels: [claudeLabel.name]
+              });
+            }

.github/workflows/label-mlops.yml

@@ -1,17 +0,0 @@
-name: Label ML Ops Team Issues
-
-on:
-  issues:
-    types:
-      - opened
-
-jobs:
-  add-mlops-label:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check if ML Ops Team is selected
-        uses: actions-ecosystem/action-add-labels@v1
-        if: contains(github.event.issue.body, '### Are you a ML Ops Team?') && contains(github.event.issue.body, 'Yes')
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          labels: "mlops user request"

.github/workflows/publish-migrations.yml

@@ -13,6 +13,7 @@ on:
 
 jobs:
   publish-migrations:
+    if: github.repository == 'BerriAI/litellm'
     runs-on: ubuntu-latest
     services:
       postgres:

.github/workflows/publish_enterprise.yml

@@ -0,0 +1,94 @@
+name: Publish litellm-enterprise to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+      pull-requests: write
+    defaults:
+      run:
+        working-directory: enterprise
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-enterprise = {version = \"${OLD}\"/litellm-enterprise = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-enterprise==${OLD}/litellm-enterprise==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump and create PR
+        id: create-pr
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          BRANCH="bump/enterprise-${{ steps.bump.outputs.new }}"
+          git checkout -b "$BRANCH"
+          git add enterprise/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push origin "$BRANCH" --force
+          gh pr create \
+            --title "bump: litellm-enterprise ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}" \
+            --body "Version bump for litellm-enterprise. Merge to update main." \
+            --head "$BRANCH" \
+            --base main \
+          || true
+          PR_URL=$(gh pr list --head "$BRANCH" --json url -q '.[0].url')
+          echo "pr_url=$PR_URL" >> $GITHUB_OUTPUT
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        run: |
+          gh pr merge "${{ steps.create-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_ENTERPRISE }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_enterprise-${{ steps.bump.outputs.new }}*

.github/workflows/publish_proxy_extras.yml

@@ -0,0 +1,74 @@
+name: Publish litellm-proxy-extras to PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      bump:
+        description: "Version bump type"
+        required: true
+        default: "patch"
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository == 'BerriAI/litellm'
+    permissions:
+      contents: write
+    defaults:
+      run:
+        working-directory: litellm-proxy-extras
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Bump version
+        id: bump
+        run: |
+          OLD=$(poetry version -s)
+          poetry version ${{ github.event.inputs.bump }}
+          NEW=$(poetry version -s)
+          echo "old=$OLD" >> $GITHUB_OUTPUT
+          echo "new=$NEW" >> $GITHUB_OUTPUT
+
+      - name: Update version refs in root pyproject.toml and requirements.txt
+        run: |
+          OLD=${{ steps.bump.outputs.old }}
+          NEW=${{ steps.bump.outputs.new }}
+          sed -i "s/litellm-proxy-extras = {version = \"${OLD}\"/litellm-proxy-extras = {version = \"${NEW}\"/" ../pyproject.toml
+          sed -i "s/litellm-proxy-extras==${OLD}/litellm-proxy-extras==${NEW}/" ../requirements.txt
+
+      - name: Update poetry.lock
+        working-directory: .
+        run: poetry lock
+
+      - name: Build
+        run: poetry build
+
+      - name: Commit version bump
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          cd ..
+          git add litellm-proxy-extras/pyproject.toml pyproject.toml requirements.txt poetry.lock
+          git commit -m "bump: litellm-proxy-extras ${{ steps.bump.outputs.old }} → ${{ steps.bump.outputs.new }}"
+          git push
+
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_PASSWORD }}
+        run: |
+          pip install twine
+          twine upload dist/litellm_proxy_extras-${{ steps.bump.outputs.new }}*

.github/workflows/regenerate-poetry-lock.yml

@@ -0,0 +1,80 @@
+name: Regenerate poetry.lock
+
+# Runs whenever pyproject.toml is merged into main (the most common cause of
+# the "pyproject.toml changed significantly since poetry.lock was last generated"
+# CI failure).  Can also be triggered manually.
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - pyproject.toml
+  workflow_dispatch:
+
+permissions:
+  contents: write       # needed to push the auto/regenerate-poetry-lock-* branch
+  pull-requests: write  # needed to open the PR and enable auto-merge
+
+jobs:
+  regenerate-lock:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install Poetry
+        run: pip install poetry
+
+      - name: Regenerate poetry.lock
+        run: poetry lock
+
+      - name: Check whether poetry.lock actually changed
+        id: diff
+        run: |
+          if git diff --quiet poetry.lock; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Open PR with the refreshed lock file
+        if: steps.diff.outputs.changed == 'true'
+        id: open-pr
+        run: |
+          BRANCH="auto/regenerate-poetry-lock-$(date +'%Y%m%d%H%M%S')"
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$BRANCH"
+          git add poetry.lock
+          git commit -m "chore: regenerate poetry.lock to match pyproject.toml"
+          git push -f origin "$BRANCH"
+
+          cat > /tmp/pr-body.md << 'BODY'
+          Automated regeneration of `poetry.lock` after `pyproject.toml` was updated on `main`.
+
+          Fixes the recurring CI failure:
+          ```
+          pyproject.toml changed significantly since poetry.lock was last generated.
+          Run `poetry lock` to fix the lock file.
+          ```
+          BODY
+
+          PR_URL=$(gh pr create \
+            --title "chore: regenerate poetry.lock to match pyproject.toml" \
+            --body-file /tmp/pr-body.md \
+            --head "$BRANCH" \
+            --base main)
+          echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Enable auto-merge
+        if: steps.diff.outputs.changed == 'true'
+        run: |
+          gh pr merge "${{ steps.open-pr.outputs.pr_url }}" --auto --squash
+        env:
+          GH_TOKEN: ${{ github.token }}

.github/workflows/run_observatory_tests.yml

@@ -0,0 +1,225 @@
+name: Run Observatory Tests
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Docker image tag to test (e.g. v1.61.0.rc1)"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash (defaults to HEAD of current branch)"
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      tag:
+        description: "Docker image tag to test"
+        required: true
+        type: string
+      commit_hash:
+        description: "Commit hash of the release"
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+env:
+  LITELLM_MASTER_KEY: ${{ secrets.LITELLM_MASTER_KEY_STAGING }}
+
+jobs:
+  observatory-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Validate tag input
+        env:
+          TAG: ${{ inputs.tag }}
+        run: |
+          if [[ ! "$TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+            echo "Invalid tag format: $TAG (expected vX.Y.Z...)"
+            exit 1
+          fi
+
+      - name: Start LiteLLM container
+        env:
+          TAG: ${{ inputs.tag }}
+          AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }}
+          AZURE_API_BASE: ${{ secrets.AZURE_API_BASE }}
+        run: |
+          docker run -d \
+            --name litellm-rc \
+            -p 4000:4000 \
+            -v "${{ github.workspace }}/.github/observatory/litellm_config.yaml:/app/config.yaml" \
+            -e LITELLM_MASTER_KEY="${LITELLM_MASTER_KEY}" \
+            -e AZURE_API_KEY="${AZURE_API_KEY}" \
+            -e AZURE_API_BASE="${AZURE_API_BASE}" \
+            "litellm/litellm:${TAG}" \
+            --config /app/config.yaml --port 4000
+
+      - name: Wait for LiteLLM health check
+        run: |
+          echo "Waiting for LiteLLM to be ready..."
+          for i in $(seq 1 30); do
+            if curl -s -f http://localhost:4000/health/liveliness > /dev/null 2>&1; then
+              echo "LiteLLM is healthy"
+              exit 0
+            fi
+            echo "Attempt $i/30 - not ready yet, waiting 10s..."
+            sleep 10
+          done
+          echo "LiteLLM failed to start within 5 minutes"
+          docker logs litellm-rc
+          exit 1
+
+      - name: Start cloudflared tunnel
+        run: |
+          # Install cloudflared
+          curl -sL https://github.com/cloudflare/cloudflared/releases/download/2025.2.1/cloudflared-linux-amd64 -o /usr/local/bin/cloudflared
+          chmod +x /usr/local/bin/cloudflared
+
+          # Start a quick tunnel (no account needed) and capture the URL
+          cloudflared tunnel --url http://localhost:4000 --no-autoupdate > /tmp/cloudflared.log 2>&1 &
+          CLOUDFLARED_PID=$!
+          echo "CLOUDFLARED_PID=$CLOUDFLARED_PID" >> $GITHUB_ENV
+
+          # Wait for tunnel URL to appear in logs
+          echo "Waiting for tunnel URL..."
+          for i in $(seq 1 30); do
+            TUNNEL_URL=$(grep -oP 'https://[a-z0-9-]+\.trycloudflare\.com' /tmp/cloudflared.log | head -1 || true)
+            if [ -n "$TUNNEL_URL" ]; then
+              echo "Tunnel URL: $TUNNEL_URL"
+              echo "TUNNEL_URL=$TUNNEL_URL" >> $GITHUB_ENV
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "Failed to get tunnel URL"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Verify tunnel connectivity
+        run: |
+          echo "Testing tunnel at ${{ env.TUNNEL_URL }}..."
+          # Quick tunnels need time for DNS propagation; retry to avoid
+          # transient NXDOMAIN (curl exit code 6) on first attempt.
+          for i in $(seq 1 10); do
+            if curl -sf "${{ env.TUNNEL_URL }}/health/liveliness" > /dev/null 2>&1; then
+              echo "Tunnel is working (attempt $i)"
+              exit 0
+            fi
+            echo "Attempt $i/10 - tunnel not routable yet, waiting 5s..."
+            sleep 5
+          done
+          echo "Tunnel failed to become reachable after 50s"
+          cat /tmp/cloudflared.log
+          exit 1
+
+      - name: Trigger observatory test run
+        id: trigger
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+        run: |
+          PAYLOAD=$(jq -n \
+            --arg url "${TUNNEL_URL}" \
+            --arg key "${LITELLM_MASTER_KEY}" \
+            '{
+              deployment_url: $url,
+              api_key: $key,
+              test_suite: "TestOAIAzureRelease",
+              models: ["gpt-4o-mini", "gpt-4o"]
+            }')
+          RESPONSE=$(curl -s -w "\n%{http_code}" -X POST "${OBSERVATORY_URL}/run-test" \
+            -H "Content-Type: application/json" \
+            -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}" \
+            -d "$PAYLOAD")
+          HTTP_CODE=$(echo "$RESPONSE" | tail -1)
+          BODY=$(echo "$RESPONSE" | head -n -1)
+          echo "Response ($HTTP_CODE): $BODY"
+          if [ "$HTTP_CODE" -ge 400 ]; then
+            echo "Failed to trigger test run"
+            exit 1
+          fi
+
+          # Extract request_id for polling this specific run
+          REQUEST_ID=$(echo "$BODY" | jq -r '.results.request_id')
+          if [ -z "$REQUEST_ID" ] || [ "$REQUEST_ID" = "null" ]; then
+            echo "Failed to extract request_id from response"
+            exit 1
+          fi
+          echo "Request ID: $REQUEST_ID"
+          echo "request_id=$REQUEST_ID" >> $GITHUB_OUTPUT
+
+      - name: Poll for test completion
+        id: poll
+        env:
+          OBSERVATORY_URL: ${{ secrets.OBSERVATORY_URL }}
+          OBSERVATORY_API_KEY: ${{ secrets.OBSERVATORY_API_KEY }}
+          REQUEST_ID: ${{ steps.trigger.outputs.request_id }}
+        run: |
+          TIMEOUT=900  # 15 minutes
+          INTERVAL=30
+          ELAPSED=0
+          while [ $ELAPSED -lt $TIMEOUT ]; do
+            STATUS=$(curl -s "${OBSERVATORY_URL}/run-status/${REQUEST_ID}" \
+              -H "X-LiteLLM-Observatory-API-Key: ${OBSERVATORY_API_KEY}")
+            RUN_STATUS=$(echo "$STATUS" | jq -r '.status')
+            echo "Run status (${ELAPSED}s elapsed): $RUN_STATUS"
+
+            if [ "$RUN_STATUS" = "completed" ] || [ "$RUN_STATUS" = "failed" ]; then
+              echo "Test finished with status: $RUN_STATUS"
+              echo "$STATUS" > /tmp/observatory_result.json
+              exit 0
+            fi
+
+            sleep $INTERVAL
+            ELAPSED=$((ELAPSED + INTERVAL))
+          done
+          echo "Timed out waiting for test to complete after ${TIMEOUT}s"
+          exit 1
+
+      - name: Verify test results
+        run: |
+          RESULT=$(cat /tmp/observatory_result.json)
+          echo "Full result: $RESULT"
+
+          STATUS=$(echo "$RESULT" | jq -r '.status')
+          TEST_PASSED=$(echo "$RESULT" | jq -r '.result.test_passed // false')
+          FAILURE_RATE=$(echo "$RESULT" | jq -r '.result.failure_rate // "N/A"')
+          ERROR=$(echo "$RESULT" | jq -r '.error // empty')
+
+          echo "Status: $STATUS"
+          echo "Test passed: $TEST_PASSED"
+          echo "Failure rate: $FAILURE_RATE"
+
+          if [ -n "$ERROR" ]; then
+            echo "Error: $ERROR"
+          fi
+
+          if [ "$STATUS" = "failed" ]; then
+            echo "Test run failed"
+            exit 1
+          fi
+
+          if [ "$TEST_PASSED" != "true" ]; then
+            echo "Tests did not pass (failure rate: $FAILURE_RATE)"
+            exit 1
+          fi
+
+          echo "All tests passed!"
+
+      - name: Print LiteLLM logs on failure
+        if: failure()
+        run: |
+          docker logs litellm-rc 2>/dev/null || true
+          cat /tmp/cloudflared.log 2>/dev/null || true
+
+      - name: Cleanup
+        if: always()
+        run: |
+          kill "${{ env.CLOUDFLARED_PID }}" 2>/dev/null || true
+          docker rm -f litellm-rc 2>/dev/null || true

.github/workflows/scan_duplicate_issues.yml

@@ -0,0 +1,47 @@
+name: Scan Duplicate Issues (One-Time)
+
+on:
+  workflow_dispatch:
+    inputs:
+      threshold:
+        description: "Similarity threshold (0-1)"
+        required: false
+        default: "0.85"
+      close:
+        description: "Actually close duplicates (false = dry run)"
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      contents: read
+    steps:
+      - name: Checkout scripts
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Scan for duplicate issues
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INPUT_THRESHOLD: ${{ inputs.threshold }}
+          INPUT_CLOSE: ${{ inputs.close }}
+        run: |
+          CLOSE_FLAG=""
+          if [ "$INPUT_CLOSE" = "true" ]; then
+            CLOSE_FLAG="--close"
+          fi
+          python3 .github/scripts/close_duplicate_issues.py \
+            --scan \
+            --repo ${{ github.repository }} \
+            --threshold "$INPUT_THRESHOLD" \
+            $CLOSE_FLAG

.github/workflows/test-linting.yml

@@ -30,13 +30,13 @@ jobs:
 
     - name: Install dependencies
       run: |
+        poetry lock
         poetry install --with dev
-        poetry run pip install openai==1.100.1
 
-    - name: Run Black formatting
+    - name: Check Black formatting
       run: |
         cd litellm
-        poetry run black .
+        poetry run black --check --exclude '/enterprise/' .
         cd ..
 
     - name: Debug - Check file state
@@ -72,4 +72,36 @@ jobs:
 
     - name: Check import safety
       run: |
-        poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+        poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+
+  secret-scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.12'
+
+    - name: Run secret scan test
+      run: |
+        pip install pytest
+        pytest tests/litellm/test_no_hardcoded_secrets.py -v
+
+    - name: Run ggshield secret scan
+      env:
+        GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
+      run: |
+        if [ -n "$GITGUARDIAN_API_KEY" ]; then
+          pip install ggshield
+          ggshield secret scan repo .
+        else
+          echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"
+        fi

.github/workflows/test-litellm-matrix.yml

@@ -0,0 +1,166 @@
+name: LiteLLM Unit Tests (Matrix)
+
+on:
+  pull_request:
+    branches: [main]
+
+# Cancel in-progress runs for the same PR
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20  # Increased from 15 to 20
+    strategy:
+      fail-fast: false
+      matrix:
+        test-group:
+          # tests/test_litellm split by subdirectory (~560 files total)
+          # Vertex AI tests separated for better isolation (prevent auth/env pollution)
+          - name: "llms-vertex"
+            path: "tests/test_litellm/llms/vertex_ai"
+            workers: 1
+            reruns: 2
+          - name: "llms-other"
+            path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+            workers: 2
+            reruns: 2
+          # tests/test_litellm/proxy split by subdirectory (~180 files total)
+          - name: "proxy-guardrails"
+            path: "tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers"
+            workers: 2
+            reruns: 2
+          - name: "proxy-core"
+            path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine"
+            workers: 2
+            reruns: 2
+          - name: "proxy-misc"
+            path: "tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py"
+            workers: 2
+            reruns: 2
+          - name: "integrations"
+            path: "tests/test_litellm/integrations"
+            workers: 2
+            reruns: 3  # Integration tests tend to be flakier
+          - name: "core-utils"
+            path: "tests/test_litellm/litellm_core_utils"
+            workers: 2
+            reruns: 1
+          - name: "other-1"
+            # responses (5942) + caching (1723) + types (819) ≈ 8.5k lines
+            path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
+            workers: 2
+            reruns: 2
+          - name: "other-2"
+            # enterprise (3062) + google_genai (2511) + router_utils (1982) ≈ 7.6k lines
+            path: "tests/test_litellm/enterprise tests/test_litellm/google_genai tests/test_litellm/router_utils"
+            workers: 2
+            reruns: 2
+          - name: "other-3"
+            # remaining dirs ≈ 8.0k lines
+            path: "tests/test_litellm/router_strategy tests/test_litellm/secret_managers tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/experimental_mcp_client tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/vector_stores"
+            workers: 2
+            reruns: 2
+          - name: "root"
+            path: "tests/test_litellm/test_*.py"
+            workers: 2
+            reruns: 2
+          # tests/proxy_unit_tests split alphabetically (~48 files total)
+          - name: "proxy-unit-a1"
+            # test_[a-j]*.py: jwt (1564) + auth_checks (978) + google_gemini (478) + e2e_pod_lock (437) + rest
+            path: "tests/proxy_unit_tests/test_[a-j]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-a2"
+            # test_[k-o]*.py: key_generate_prisma (4346) + key_generate_dynamodb + models_fallback
+            path: "tests/proxy_unit_tests/test_[k-o]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b1"
+            # lighter config/utility proxy tests (prisma, project, prompt, proxy_[c-r]*)
+            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_project*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b2"
+            # proxy_server.py alone (2750 lines) - isolated to avoid blocking smaller tests
+            path: "tests/proxy_unit_tests/test_proxy_server.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b3"
+            # proxy_server_* (618) + proxy_setting_guardrails (71) - smaller server-related tests
+            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b4"
+            # proxy_utils.py alone (2339 lines) - isolated to avoid blocking token counter
+            path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b5"
+            # proxy_token_counter (1279) - runs independently from utils
+            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b6"
+            # test_[r-t]*.py: response_polling (1399) + search_api_logging (202) + server_root (64) + skills_db (261) + realtime_cache (62)
+            path: "tests/proxy_unit_tests/test_[r-t]*.py"
+            workers: 2
+            reruns: 1
+          - name: "proxy-unit-b7"
+            # test_[u-z]*.py: user_api_key_auth (1136) + zero_cost (590) + update_spend (305) + unit_test_* (206) + ui_path (157)
+            path: "tests/proxy_unit_tests/test_[u-z]*.py"
+            workers: 2
+            reruns: 1
+
+    name: test (${{ matrix.test-group.name }})
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy semantic-router"
+          # pytest-rerunfailures and pytest-xdist are in pyproject.toml dev dependencies
+          poetry run pip install google-genai==1.22.0 \
+            google-cloud-aiplatform>=1.38 fastapi-offline==1.7.3 python-multipart==0.0.22 openapi-core
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        run: |
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run tests - ${{ matrix.test-group.name }}
+        run: |
+          poetry run pytest ${{ matrix.test-group.path }} \
+            --tb=short -vv \
+            --maxfail=10 \
+            -n ${{ matrix.test-group.workers }} \
+            --reruns ${{ matrix.test-group.reruns }} \
+            --reruns-delay 1 \
+            --dist=loadscope \
+            --durations=20

.github/workflows/test-litellm-ui-build.yml

@@ -0,0 +1,32 @@
+name: UI Build Check
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  build-ui:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: ui/litellm-dashboard
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+          cache-dependency-path: ui/litellm-dashboard/package-lock.json
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Build
+        run: npm run build

.github/workflows/test-litellm.yml

@@ -1,8 +1,12 @@
 name: LiteLLM Mock Tests (folder - tests/test_litellm)
 
+# DEPRECATED: This workflow is replaced by test-litellm-matrix.yml which runs
+# the same tests in parallel across 10 jobs for faster CI times.
+# Kept for manual debugging only.
 on:
-  pull_request:
-    branches: [ main ]
+  workflow_dispatch:  # Manual trigger only
+  # pull_request:
+  #   branches: [ main ]
 
 jobs:
   test:
@@ -27,18 +31,18 @@ jobs:
 
     - name: Install dependencies
       run: |
+        poetry lock
         poetry install --with dev,proxy-dev --extras "proxy semantic-router"
         poetry run pip install "pytest-retry==1.6.3"
         poetry run pip install pytest-xdist
         poetry run pip install "google-genai==1.22.0"
         poetry run pip install "google-cloud-aiplatform>=1.38"
         poetry run pip install "fastapi-offline==1.7.3"
-        poetry run pip install "python-multipart==0.0.18"
+        poetry run pip install "python-multipart>=0.0.20"
+        poetry run pip install "openapi-core"
     - name: Setup litellm-enterprise as local package
       run: |
-        cd enterprise
-        python -m pip install -e .
-        cd ..
+        poetry run pip install --force-reinstall --no-deps -e enterprise/
     - name: Run tests
       run: |
         poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4 --durations=50

.github/workflows/test-mcp.yml

@@ -27,21 +27,20 @@ jobs:
 
     - name: Install dependencies
       run: |
+        poetry lock
         poetry install --with dev,proxy-dev --extras "proxy semantic-router"
         poetry run pip install "pytest==7.3.1"
         poetry run pip install "pytest-retry==1.6.3"
         poetry run pip install "pytest-cov==5.0.0"
         poetry run pip install "pytest-asyncio==0.21.1"
         poetry run pip install "respx==0.22.0"
-        poetry run pip install "pydantic==2.10.2"
-        poetry run pip install "mcp==1.10.1"
+        poetry run pip install "pydantic==2.11.0"
+        poetry run pip install "mcp==1.25.0"
         poetry run pip install pytest-xdist
 
     - name: Setup litellm-enterprise as local package
       run: |
-        cd enterprise
-        python -m pip install -e .
-        cd ..
+        poetry run pip install --force-reinstall --no-deps -e enterprise/
 
     - name: Run MCP tests
       run: |

.github/workflows/test-model-map.yaml

@@ -0,0 +1,15 @@
+name: Validate model_prices_and_context_window.json
+
+on:
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  validate-model-prices-json:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Validate model_prices_and_context_window.json
+        run: |
+          jq empty model_prices_and_context_window.json

.github/workflows/test-proxy-e2e-azure-batches.yml

@@ -0,0 +1,90 @@
+name: Proxy E2E Azure Batches Tests
+
+on:
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  proxy_e2e_azure_batches_tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+            .venv
+          key: ${{ runner.os }}-poetry-e2e-batches-${{ hashFiles('poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-e2e-batches-
+            ${{ runner.os }}-poetry-
+
+      - name: Install dependencies
+        run: |
+          poetry config virtualenvs.in-project true
+          poetry install --with dev,proxy-dev --extras "proxy"
+          poetry run pip install psycopg2-binary uvicorn fastapi httpx tenacity
+
+      - name: Setup litellm-enterprise
+        run: |
+          poetry run pip install --force-reinstall --no-deps -e enterprise/
+
+      - name: Generate Prisma client
+        run: |
+          poetry run prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Run Prisma migrations
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+        run: |
+          cd litellm/proxy
+          poetry run prisma migrate deploy --schema schema.prisma
+          cd ../..
+
+      - name: Run Azure Batch E2E Tests
+        env:
+          DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+          USE_LOCAL_LITELLM: "true"
+          USE_MOCK_MODELS: "true"
+          USE_STATE_TRACKER: "true"
+          LITELLM_LOG: DEBUG
+        run: |
+          poetry run pytest tests/proxy_e2e_azure_batches_tests/test_proxy_e2e_azure_batches.py \
+            -vv -s -k "test_e2e_managed_batch" \
+            --tb=short \
+            --maxfail=3 \
+            --durations=10
+

.github/workflows/test_server_root_path.yml

@@ -0,0 +1,96 @@
+name: Test Proxy SERVER_ROOT_PATH Routing
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  test-server-root-path:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    strategy:
+      matrix:
+        root_path: ["/api/v1", "/llmproxy"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./docker/Dockerfile.non_root
+          tags: litellm-test:${{ github.sha }}
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Start LiteLLM container with SERVER_ROOT_PATH
+        run: |
+          docker run -d \
+            --name litellm-test \
+            -p 4000:4000 \
+            -e SERVER_ROOT_PATH="${{ matrix.root_path }}" \
+            -e LITELLM_MASTER_KEY="sk-1234" \
+            litellm-test:${{ github.sha }} \
+            --detailed_debug
+
+      - name: Wait for container to be healthy
+        run: |
+          echo "Waiting for LiteLLM to start..."
+          max_attempts=30
+          attempt=0
+
+          while [ $attempt -lt $max_attempts ]; do
+            if docker logs litellm-test 2>&1 | grep -q "Uvicorn running"; then
+              echo "LiteLLM started successfully"
+              break
+            fi
+            attempt=$((attempt + 1))
+            echo "Attempt $attempt/$max_attempts - waiting for server to start..."
+            sleep 2
+          done
+
+          if [ $attempt -eq $max_attempts ]; then
+            echo "Server failed to start within timeout"
+            docker logs litellm-test
+            exit 1
+          fi
+
+          sleep 5
+
+      - name: Show container logs
+        if: always()
+        run: docker logs litellm-test
+
+      - name: Test UI endpoint with root path
+        run: |
+          ROOT_PATH="${{ matrix.root_path }}"
+          echo "Testing UI at: http://localhost:4000${ROOT_PATH}/ui/"
+
+          for i in 1 2 3; do
+            content=$(curl -sL --max-time 5 -H "Authorization: Bearer sk-1234" "http://localhost:4000${ROOT_PATH}/ui/")
+            if echo "$content" | grep -q -E "(html|<!DOCTYPE|<head|<body)"; then
+              echo "UI page contains valid HTML content"
+              exit 0
+            fi
+            echo "Attempt $i/3 - no valid HTML, retrying in 5s..."
+            sleep 5
+          done
+          echo "UI page does not contain expected HTML content"
+          echo "Response: $content"
+          docker logs litellm-test
+          exit 1
+
+      - name: Cleanup
+        if: always()
+        run: |
+          docker stop litellm-test || true
+          docker rm litellm-test || true

.gitignore

@@ -1,6 +1,8 @@
 .python-version
 .venv
+.venv_policy_test
 .env
+.claude
 .newenv
 newenv/*
 litellm/proxy/myenv/*
@@ -59,9 +61,6 @@ litellm/proxy/_super_secret_config.yaml
 litellm/proxy/myenv/bin/activate
 litellm/proxy/myenv/bin/Activate.ps1
 myenv/*
-litellm/proxy/_experimental/out/404/index.html
-litellm/proxy/_experimental/out/model_hub/index.html
-litellm/proxy/_experimental/out/onboarding/index.html
 litellm/tests/log.txt
 litellm/tests/langfuse.log
 litellm/tests/langfuse.log
@@ -74,9 +73,6 @@ tests/local_testing/log.txt
 litellm/proxy/_new_new_secret_config.yaml
 litellm/proxy/custom_guardrail.py
 .mypy_cache/*
-litellm/proxy/_experimental/out/404.html
-litellm/proxy/_experimental/out/404.html
-litellm/proxy/_experimental/out/model_hub.html
 .mypy_cache/*
 litellm/proxy/application.log
 tests/llm_translation/vertex_test_account.json
@@ -93,10 +89,16 @@ tests/test_custom_dir/*
 test.py
 
 litellm_config.yaml
+!.github/observatory/litellm_config.yaml
 .cursor
 .vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*
 update_model_cost_map.py
 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
-litellm/proxy/_experimental/out/guardrails/index.html
 scripts/test_vertex_ai_search.py
+LAZY_LOADING_IMPROVEMENTS.md
+STABILIZATION_TODO.md
+**/test-results
+**/playwright-report
+**/*.storageState.json
+**/coverage

.semgrep/rules/README.md

@@ -0,0 +1,22 @@
+# Custom Semgrep rules for LiteLLM
+
+Add custom rule YAML files here. Semgrep loads all `.yml`/`.yaml` files under this directory.
+
+**Run only custom rules (CI / fail on findings):**
+
+```bash
+semgrep scan --config .semgrep/rules . --error
+```
+
+**Run with registry + custom rules:**
+
+```bash
+semgrep scan --config auto --config .semgrep/rules .
+```
+
+**Layout:**
+
+- `python/` – Python-specific rules (security, patterns)
+- Add more subdirs as needed (e.g. `generic/` for language-agnostic rules)
+
+See [Semgrep rule syntax](https://semgrep.dev/docs/writing-rules/rule-syntax/).

.semgrep/rules/python/reliability/unbounded-memory.yml

@@ -0,0 +1,17 @@
+# Unbounded memory growth – data structures without a clear max limit
+# Can lead to OOM under load.
+
+rules:
+  - id: unbounded-asyncio-queue
+    message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues).
+    severity: ERROR
+    languages: [python]
+    pattern-either:
+      - pattern: asyncio.Queue()
+      - pattern: asyncio.Queue(maxsize=0)
+    metadata:
+      category: reliability
+      cwe: "CWE-400: Uncontrolled Resource Consumption"
+      tags: [python, reliability]
+      confidence: HIGH
+      source: https://docs.python.org/3/library/asyncio-queue.html

.semgrep/rules/python/unbounded-memory.yml

@@ -0,0 +1,14 @@
+# Unbounded memory growth – data structures without a clear max limit
+# Can lead to OOM under load.
+
+rules:
+  - id: unbounded-asyncio-queue
+    message: asyncio.Queue() with no maxsize can grow unbounded. Use asyncio.Queue(maxsize=N) for integrations (e.g. log queues).
+    severity: ERROR
+    languages: [python]
+    pattern-either:
+      - pattern: asyncio.Queue()
+      - pattern: asyncio.Queue(maxsize=0)
+    metadata:
+      category: correctness
+      cwe: "CWE-400: Uncontrolled Resource Consumption"

.trivyignore

@@ -0,0 +1,12 @@
+# LiteLLM Trivy Ignore File
+# CVEs listed here are temporarily allowlisted pending fixes
+
+# Next.js vulnerabilities in UI dashboard (next@14.2.35)
+# Allowlisted: 2026-01-31, 7-day fix timeline
+# Fix: Upgrade to Next.js 15.5.10+ or 16.1.5+
+
+# HIGH: DoS via request deserialization
+GHSA-h25m-26qc-wcjf
+
+# MEDIUM: Image Optimizer DoS
+CVE-2025-59471

AGENTS.md

@@ -49,6 +49,29 @@ LiteLLM is a unified interface for 100+ LLMs that:
    - Test provider-specific functionality thoroughly
    - Consider adding load tests for performance-critical changes
 
+### MAKING CODE CHANGES FOR THE UI (IGNORE FOR BACKEND)
+
+1. **Tremor is DEPRECATED, do not use Tremor components in new features/changes**
+   - The only exception is the Tremor Table component and its required Tremor Table sub components.
+
+2. **Use Common Components as much as possible**:
+   - These are usually defined in the `common_components` directory
+   - Use these components as much as possible and avoid building new components unless needed
+
+3. **Testing**:
+   - The codebase uses **Vitest** and **React Testing Library**
+   - **Query Priority Order**: Use query methods in this order: `getByRole`, `getByLabelText`, `getByPlaceholderText`, `getByText`, `getByTestId`
+   - **Always use `screen`** instead of destructuring from `render()` (e.g., use `screen.getByText()` not `getByText`)
+   - **Wrap user interactions in `act()`**: Always wrap `fireEvent` calls with `act()` to ensure React state updates are properly handled
+   - **Use `query` methods for absence checks**: Use `queryBy*` methods (not `getBy*`) when expecting an element to NOT be present
+   - **Test names must start with "should"**: All test names should follow the pattern `it("should ...")`
+   - **Mock external dependencies**: Check `setupTests.ts` for global mocks and mock child components/networking calls as needed
+   - **Structure tests properly**:
+     - First test should verify the component renders successfully
+     - Subsequent tests should focus on functionality and user interactions
+     - Use `waitFor` for async operations that aren't already awaited
+   - **Avoid using `querySelector`**: Prefer React Testing Library queries over direct DOM manipulation
+
 ### IMPORTANT PATTERNS
 
 1. **Function/Tool Calling**:
@@ -86,6 +109,8 @@ Key files:
 - `litellm/proxy/auth/` - Authentication logic
 - `litellm/proxy/management_endpoints/` - Admin API endpoints
 
+**Database (proxy)**: Use Prisma model methods (`prisma_client.db.<model>.upsert`, `.find_many`, `.find_unique`, etc.), not raw SQL (`execute_raw`/`query_raw`). See COMMON PITFALLS for details.
+
 ## MCP (MODEL CONTEXT PROTOCOL) SUPPORT
 
 LiteLLM supports MCP for agent workflows:
@@ -94,6 +119,29 @@ LiteLLM supports MCP for agent workflows:
 - Support for external MCP servers (Zapier, Jira, Linear, etc.)
 - See `litellm/experimental_mcp_client/` and `litellm/proxy/_experimental/mcp_server/`
 
+## RUNNING SCRIPTS
+
+Use `poetry run python script.py` to run Python scripts in the project environment (for non-test files).
+
+## GITHUB TEMPLATES
+
+When opening issues or pull requests, follow these templates:
+
+### Bug Reports (`.github/ISSUE_TEMPLATE/bug_report.yml`)
+- Describe what happened vs. expected behavior
+- Include relevant log output
+- Specify LiteLLM version
+- Indicate if you're part of an ML Ops team (helps with prioritization)
+
+### Feature Requests (`.github/ISSUE_TEMPLATE/feature_request.yml`)
+- Clearly describe the feature
+- Explain motivation and use case with concrete examples
+
+### Pull Requests (`.github/pull_request_template.md`)
+- Add at least 1 test in `tests/litellm/`
+- Ensure `make test-unit` passes
+
+
 ## TESTING CONSIDERATIONS
 
 1. **Provider Tests**: Test against real provider APIs when possible
@@ -128,6 +176,43 @@ LiteLLM supports MCP for agent workflows:
 3. **Rate Limits**: Respect provider rate limits in tests
 4. **Memory Usage**: Be mindful of memory usage in streaming scenarios
 5. **Dependencies**: Keep dependencies minimal and well-justified
+6. **UI/Backend Contract Mismatch**: When adding a new entity type to the UI, always check whether the backend endpoint accepts a single value or an array. Match the UI control accordingly (single-select vs. multi-select) to avoid silently dropping user selections
+7. **Missing Tests for New Entity Types**: When adding a new entity type (e.g., in `EntityUsage`, `UsageViewSelect`), always add corresponding tests in the existing test files and update any icon/component mocks
+8. **Raw SQL in proxy DB code**: Do not use `execute_raw` or `query_raw` for proxy database access. Use Prisma model methods (e.g. `prisma_client.db.litellm_tooltable.upsert()`, `.find_many()`, `.find_unique()`) so behavior stays consistent with the schema, the client stays mockable in tests, and you avoid the pitfalls of hand-written SQL (parameter ordering, type casting, schema drift)
+
+8. **Do not hardcode model-specific flags**: Put model-specific capability flags in `model_prices_and_context_window.json` and read them via `get_model_info` (or existing helpers like `supports_reasoning`). This prevents users from needing to upgrade LiteLLM each time a new model supports a feature.
+
+   **Example of BAD** (hardcoded model checks):
+
+   ```python
+   @staticmethod
+   def _is_effort_supported_model(model: str) -> bool:
+       """Check if the model supports the output_config.effort parameter..."""
+       model_lower = model.lower()
+       if AnthropicConfig._is_claude_4_6_model(model):
+           return True
+       return any(
+           v in model_lower for v in ("opus-4-5", "opus_4_5", "opus-4.5", "opus_4.5")
+       )
+   ```
+
+   **Example of GOOD** (config-driven or helper that reads from config):
+
+   ```python
+   if (
+       "claude-3-7-sonnet" in model
+       or AnthropicConfig._is_claude_4_6_model(model)
+       or supports_reasoning(
+           model=model,
+           custom_llm_provider=self.custom_llm_provider,
+       )
+   ):
+       ...
+   ```
+
+   Using helpers like `supports_reasoning` (which read from `model_prices_and_context_window.json` / `get_model_info`) allows future model updates to "just work" without code changes.
+
+9. **Never close HTTP/SDK clients on cache eviction**: Do not add `close()`, `aclose()`, or `create_task(close_fn())` inside `LLMClientCache._remove_key()` or any cache eviction path. Evicted clients may still be held by in-flight requests; closing them causes `RuntimeError: Cannot send a request, as the client has been closed.` in production after the cache TTL (1 hour) expires. Connection cleanup is handled at shutdown by `close_litellm_async_clients()`. See PR #22247 for the full incident history.
 
 ## HELPFUL RESOURCES
 
@@ -141,4 +226,49 @@ LiteLLM supports MCP for agent workflows:
 - Check similar provider implementations
 - Ensure comprehensive test coverage
 - Update documentation appropriately
-- Consider backward compatibility impact 
+- Consider backward compatibility impact
+
+## Cursor Cloud specific instructions
+
+### Environment
+
+- Poetry is installed in `~/.local/bin`; the update script ensures it is on `PATH`.
+- Python 3.12, Node 22 are pre-installed.
+- The virtual environment lives under `~/.cache/pypoetry/virtualenvs/`.
+
+### Running the proxy server
+
+Start the proxy with a config file:
+
+```bash
+poetry run litellm --config dev_config.yaml --port 4000
+```
+
+The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.
+
+### Running tests
+
+See `CLAUDE.md` and the `Makefile` for standard commands. Key notes:
+
+- `psycopg-binary` must be installed (`poetry run pip install psycopg-binary`) because the pytest-postgresql plugin requires it and the lock file only includes `psycopg` (no binary).
+- `openapi-core` must be installed (`poetry run pip install openapi-core`) for the OpenAPI compliance tests in `tests/test_litellm/interactions/`.
+- The `--timeout` pytest flag is NOT available; don't pass it.
+- Unit tests: `poetry run pytest tests/test_litellm/ -x -vv -n 4`
+- Black `--check` may report pre-existing formatting issues; this does not block test runs.
+- If `poetry install` fails with "pyproject.toml changed significantly since poetry.lock was last generated", run `poetry lock` first to regenerate the lock file.
+
+### Lint
+
+```bash
+cd litellm && poetry run ruff check .
+```
+
+Ruff is the primary fast linter. For the full lint suite (including mypy, black, circular imports), run `make lint` per `CLAUDE.md`.
+
+### UI Dashboard development
+
+- The UI is at `ui/litellm-dashboard/`. Run `npm run dev` from that directory for the Next.js dev server on port 3000.
+- The proxy at port 4000 serves a **pre-built** static UI from `litellm/proxy/_experimental/out/`. After making UI code changes, you must run `npm run build` in the dashboard directory and copy the output: `cp -r ui/litellm-dashboard/out/* litellm/proxy/_experimental/out/` for the proxy to serve the updated UI.
+- SVGs used as provider logos (loaded via `<img>` tags) must NOT use `fill="currentColor"` — replace with an explicit color like `#000000` or use the `-color` variant from lobehub icons, since CSS color inheritance does not work inside `<img>` elements.
+- Provider logos live in `ui/litellm-dashboard/public/assets/logos/` (source) and `litellm/proxy/_experimental/out/assets/logos/` (pre-built). Both locations must have the file for it to work in dev and proxy-served modes.
+- UI Vitest tests: `cd ui/litellm-dashboard && npx vitest run`

ARCHITECTURE.md

@@ -0,0 +1,398 @@
+# LiteLLM Architecture - LiteLLM SDK + AI Gateway
+
+This document helps contributors understand where to make changes in LiteLLM.
+
+---
+
+## How It Works
+
+The LiteLLM AI Gateway (Proxy) uses the LiteLLM SDK internally for all LLM calls:
+
+```
+OpenAI SDK (client)    ──▶  LiteLLM AI Gateway (proxy/)  ──▶  LiteLLM SDK (litellm/)  ──▶  LLM API
+Anthropic SDK (client) ──▶  LiteLLMAI Gateway (proxy/)  ──▶  LiteLLM SDK (litellm/)  ──▶  LLM API
+Any HTTP client        ──▶  LiteLLMAI Gateway (proxy/)  ──▶  LiteLLM SDK (litellm/)  ──▶  LLM API
+```
+
+The **AI Gateway** adds authentication, rate limiting, budgets, and routing on top of the SDK.
+The **SDK** handles the actual LLM provider calls, request/response transformations, and streaming.
+
+---
+
+## 1. AI Gateway (Proxy) Request Flow
+
+The AI Gateway (`litellm/proxy/`) wraps the SDK with authentication, rate limiting, and management features.
+
+```mermaid
+sequenceDiagram
+    participant Client
+    participant ProxyServer as proxy/proxy_server.py
+    participant Auth as proxy/auth/user_api_key_auth.py
+    participant Redis as Redis Cache
+    participant Hooks as proxy/hooks/
+    participant Router as router.py
+    participant Main as main.py + utils.py
+    participant Handler as llms/custom_httpx/llm_http_handler.py
+    participant Transform as llms/{provider}/chat/transformation.py
+    participant Provider as LLM Provider API
+    participant CostCalc as cost_calculator.py
+    participant LoggingObj as litellm_logging.py
+    participant DBWriter as db/db_spend_update_writer.py
+    participant Postgres as PostgreSQL
+
+    %% Request Flow
+    Client->>ProxyServer: POST /v1/chat/completions
+    ProxyServer->>Auth: user_api_key_auth()
+    Auth->>Redis: Check API key cache
+    Redis-->>Auth: Key info + spend limits
+    ProxyServer->>Hooks: max_budget_limiter, parallel_request_limiter
+    Hooks->>Redis: Check/increment rate limit counters
+    ProxyServer->>Router: route_request()
+    Router->>Main: litellm.acompletion()
+    Main->>Handler: BaseLLMHTTPHandler.completion()
+    Handler->>Transform: ProviderConfig.transform_request()
+    Handler->>Provider: HTTP Request
+    Provider-->>Handler: Response
+    Handler->>Transform: ProviderConfig.transform_response()
+    Transform-->>Handler: ModelResponse
+    Handler-->>Main: ModelResponse
+    
+    %% Cost Attribution (in utils.py wrapper)
+    Main->>LoggingObj: update_response_metadata()
+    LoggingObj->>CostCalc: _response_cost_calculator()
+    CostCalc->>CostCalc: completion_cost(tokens × price)
+    CostCalc-->>LoggingObj: response_cost
+    LoggingObj-->>Main: Set response._hidden_params["response_cost"]
+    Main-->>ProxyServer: ModelResponse (with cost in _hidden_params)
+    
+    %% Response Headers + Async Logging
+    ProxyServer->>ProxyServer: Extract cost from hidden_params
+    ProxyServer->>LoggingObj: async_success_handler()
+    LoggingObj->>Hooks: async_log_success_event()
+    Hooks->>DBWriter: update_database(response_cost)
+    DBWriter->>Redis: Queue spend increment
+    DBWriter->>Postgres: Batch write spend logs (async)
+    ProxyServer-->>Client: ModelResponse + x-litellm-response-cost header
+```
+
+### Proxy Components
+
+```mermaid
+graph TD
+    subgraph "Incoming Request"
+        Client["POST /v1/chat/completions"]
+    end
+
+    subgraph "proxy/proxy_server.py"
+        Endpoint["chat_completion()"]
+    end
+
+    subgraph "proxy/auth/"
+        Auth["user_api_key_auth()"]
+    end
+
+    subgraph "proxy/"
+        PreCall["litellm_pre_call_utils.py"]
+        RouteRequest["route_llm_request.py"]
+    end
+
+    subgraph "litellm/"
+        Router["router.py"]
+        Main["main.py"]
+    end
+
+    subgraph "Infrastructure"
+        DualCache["DualCache<br/>(in-memory + Redis)"]
+        Postgres["PostgreSQL<br/>(keys, teams, spend logs)"]
+    end
+
+    Client --> Endpoint
+    Endpoint --> Auth
+    Auth --> DualCache
+    DualCache -.->|cache miss| Postgres
+    Auth --> PreCall
+    PreCall --> RouteRequest
+    RouteRequest --> Router
+    Router --> DualCache
+    Router --> Main
+    Main --> Client
+```
+
+**Key proxy files:**
+- `proxy/proxy_server.py` - Main API endpoints
+- `proxy/auth/` - Authentication (API keys, JWT, OAuth2)
+- `proxy/hooks/` - Proxy-level callbacks
+- `router.py` - Load balancing, fallbacks
+- `router_strategy/` - Routing algorithms (`lowest_latency.py`, `simple_shuffle.py`, etc.)
+
+**LLM-specific proxy endpoints:**
+
+| Endpoint | Directory | Purpose |
+|----------|-----------|---------|
+| `/v1/messages` | `proxy/anthropic_endpoints/` | Anthropic Messages API |
+| `/vertex-ai/*` | `proxy/vertex_ai_endpoints/` | Vertex AI passthrough |
+| `/gemini/*` | `proxy/google_endpoints/` | Google AI Studio passthrough |
+| `/v1/images/*` | `proxy/image_endpoints/` | Image generation |
+| `/v1/batches` | `proxy/batches_endpoints/` | Batch processing |
+| `/v1/files` | `proxy/openai_files_endpoints/` | File uploads |
+| `/v1/fine_tuning` | `proxy/fine_tuning_endpoints/` | Fine-tuning jobs |
+| `/v1/rerank` | `proxy/rerank_endpoints/` | Reranking |
+| `/v1/responses` | `proxy/response_api_endpoints/` | OpenAI Responses API |
+| `/v1/vector_stores` | `proxy/vector_store_endpoints/` | Vector stores |
+| `/*` (passthrough) | `proxy/pass_through_endpoints/` | Direct provider passthrough |
+
+**Proxy Hooks** (`proxy/hooks/__init__.py`):
+
+| Hook | File | Purpose |
+|------|------|---------|
+| `max_budget_limiter` | `proxy/hooks/max_budget_limiter.py` | Enforce budget limits |
+| `parallel_request_limiter` | `proxy/hooks/parallel_request_limiter_v3.py` | Rate limiting per key/user |
+| `cache_control_check` | `proxy/hooks/cache_control_check.py` | Cache validation |
+| `responses_id_security` | `proxy/hooks/responses_id_security.py` | Response ID validation |
+| `litellm_skills` | `proxy/hooks/skills_injection.py` | Skills injection |
+
+To add a new proxy hook, implement `CustomLogger` and register in `PROXY_HOOKS`.
+
+### Infrastructure Components
+
+The AI Gateway uses external infrastructure for persistence and caching:
+
+```mermaid
+graph LR
+    subgraph "AI Gateway (proxy/)"
+        Proxy["proxy_server.py"]
+        Auth["auth/user_api_key_auth.py"]
+        DBWriter["db/db_spend_update_writer.py<br/>DBSpendUpdateWriter"]
+        InternalCache["utils.py<br/>InternalUsageCache"]
+        CostCallback["hooks/proxy_track_cost_callback.py<br/>_ProxyDBLogger"]
+        Scheduler["APScheduler<br/>ProxyStartupEvent"]
+    end
+
+    subgraph "SDK (litellm/)"
+        Router["router.py<br/>Router.cache (DualCache)"]
+        LLMCache["caching/caching_handler.py<br/>LLMCachingHandler"]
+        CacheClass["caching/caching.py<br/>Cache"]
+    end
+
+    subgraph "Redis (caching/redis_cache.py)"
+        RateLimit["Rate Limit Counters"]
+        SpendQueue["Spend Increment Queue"]
+        KeyCache["API Key Cache"]
+        TPM_RPM["TPM/RPM Tracking"]
+        Cooldowns["Deployment Cooldowns"]
+        LLMResponseCache["LLM Response Cache"]
+    end
+
+    subgraph "PostgreSQL (proxy/schema.prisma)"
+        Keys["LiteLLM_VerificationToken"]
+        Teams["LiteLLM_TeamTable"]
+        SpendLogs["LiteLLM_SpendLogs"]
+        Users["LiteLLM_UserTable"]
+    end
+
+    Auth --> InternalCache
+    InternalCache --> KeyCache
+    InternalCache -.->|cache miss| Keys
+    InternalCache --> RateLimit
+    Router --> TPM_RPM
+    Router --> Cooldowns
+    LLMCache --> CacheClass
+    CacheClass --> LLMResponseCache
+    CostCallback --> DBWriter
+    DBWriter --> SpendQueue
+    DBWriter --> SpendLogs
+    Scheduler --> SpendLogs
+    Scheduler --> Keys
+```
+
+| Component | Purpose | Key Files/Classes |
+|-----------|---------|-------------------|
+| **Redis** | Rate limiting, API key caching, TPM/RPM tracking, cooldowns, LLM response caching, spend queuing | `caching/redis_cache.py` (`RedisCache`), `caching/dual_cache.py` (`DualCache`) |
+| **PostgreSQL** | API keys, teams, users, spend logs | `proxy/utils.py` (`PrismaClient`), `proxy/schema.prisma` |
+| **InternalUsageCache** | Proxy-level cache for rate limits + API keys (in-memory + Redis) | `proxy/utils.py` (`InternalUsageCache`) |
+| **Router.cache** | TPM/RPM tracking, deployment cooldowns, client caching (in-memory + Redis) | `router.py` (`Router.cache: DualCache`) |
+| **LLMCachingHandler** | SDK-level LLM response/embedding caching | `caching/caching_handler.py` (`LLMCachingHandler`), `caching/caching.py` (`Cache`) |
+| **DBSpendUpdateWriter** | Batches spend updates to reduce DB writes | `proxy/db/db_spend_update_writer.py` (`DBSpendUpdateWriter`) |
+| **Cost Tracking** | Calculates and logs response costs | `proxy/hooks/proxy_track_cost_callback.py` (`_ProxyDBLogger`) |
+
+**Background Jobs** (APScheduler, initialized in `proxy/proxy_server.py` → `ProxyStartupEvent.initialize_scheduled_background_jobs()`):
+
+| Job | Interval | Purpose | Key Files |
+|-----|----------|---------|-----------|
+| `update_spend` | 60s | Batch write spend logs to PostgreSQL | `proxy/db/db_spend_update_writer.py` |
+| `reset_budget` | 10-12min | Reset budgets for keys/users/teams | `proxy/management_helpers/budget_reset_job.py` |
+| `add_deployment` | 10s | Sync new model deployments from DB | `proxy/proxy_server.py` (`ProxyConfig`) |
+| `cleanup_old_spend_logs` | cron/interval | Delete old spend logs | `proxy/management_helpers/spend_log_cleanup.py` |
+| `check_batch_cost` | 30min | Calculate costs for batch jobs | `proxy/management_helpers/check_batch_cost_job.py` |
+| `check_responses_cost` | 30min | Calculate costs for responses API | `proxy/management_helpers/check_responses_cost_job.py` |
+| `process_rotations` | 1hr | Auto-rotate API keys | `proxy/management_helpers/key_rotation_manager.py` |
+| `_run_background_health_check` | continuous | Health check model deployments | `proxy/proxy_server.py` |
+| `send_weekly_spend_report` | weekly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
+| `send_monthly_spend_report` | monthly | Slack spend alerts | `proxy/utils.py` (`SlackAlerting`) |
+
+**Cost Attribution Flow:**
+1. LLM response returns to `utils.py` wrapper after `litellm.acompletion()` completes
+2. `update_response_metadata()` (`llm_response_utils/response_metadata.py`) is called
+3. `logging_obj._response_cost_calculator()` (`litellm_logging.py`) calculates cost via `litellm.completion_cost()` (`cost_calculator.py`)
+4. Cost is stored in `response._hidden_params["response_cost"]`
+5. `proxy/common_request_processing.py` extracts cost from `hidden_params` and adds to response headers (`x-litellm-response-cost`)
+6. `logging_obj.async_success_handler()` triggers callbacks including `_ProxyDBLogger.async_log_success_event()`
+7. `DBSpendUpdateWriter.update_database()` queues spend increments to Redis
+8. Background job `update_spend` flushes queued spend to PostgreSQL every 60s
+
+---
+
+## 2. SDK Request Flow
+
+The SDK (`litellm/`) provides the core LLM calling functionality used by both direct SDK users and the AI Gateway.
+
+```mermaid
+graph TD
+    subgraph "SDK Entry Points"
+        Completion["litellm.completion()"]
+        Messages["litellm.messages()"]
+    end
+
+    subgraph "main.py"
+        Main["completion()<br/>acompletion()"]
+    end
+
+    subgraph "utils.py"
+        GetProvider["get_llm_provider()"]
+    end
+
+    subgraph "llms/custom_httpx/"
+        Handler["llm_http_handler.py<br/>BaseLLMHTTPHandler"]
+        HTTP["http_handler.py<br/>HTTPHandler / AsyncHTTPHandler"]
+    end
+
+    subgraph "llms/{provider}/chat/"
+        TransformReq["transform_request()"]
+        TransformResp["transform_response()"]
+    end
+
+    subgraph "litellm_core_utils/"
+        Streaming["streaming_handler.py"]
+    end
+
+    subgraph "integrations/ (async, off main thread)"
+        Callbacks["custom_logger.py<br/>Langfuse, Datadog, etc."]
+    end
+
+    Completion --> Main
+    Messages --> Main
+    Main --> GetProvider
+    GetProvider --> Handler
+    Handler --> TransformReq
+    TransformReq --> HTTP
+    HTTP --> Provider["LLM Provider API"]
+    Provider --> HTTP
+    HTTP --> TransformResp
+    TransformResp --> Streaming
+    Streaming --> Response["ModelResponse"]
+    Response -.->|async| Callbacks
+```
+
+**Key SDK files:**
+- `main.py` - Entry points: `completion()`, `acompletion()`, `embedding()`
+- `utils.py` - `get_llm_provider()` resolves model → provider
+- `llms/custom_httpx/llm_http_handler.py` - Central HTTP orchestrator
+- `llms/custom_httpx/http_handler.py` - Low-level HTTP client
+- `llms/{provider}/chat/transformation.py` - Provider-specific transformations
+- `litellm_core_utils/streaming_handler.py` - Streaming response handling
+- `integrations/` - Async callbacks (Langfuse, Datadog, etc.)
+
+---
+
+## 3. Translation Layer
+
+When a request comes in, it goes through a **translation layer** that converts between API formats.
+Each translation is isolated in its own file, making it easy to test and modify independently.
+
+### Where to find translations
+
+| Incoming API | Provider | Translation File |
+|--------------|----------|------------------|
+| `/v1/chat/completions` | Anthropic | `llms/anthropic/chat/transformation.py` |
+| `/v1/chat/completions` | Bedrock Converse | `llms/bedrock/chat/converse_transformation.py` |
+| `/v1/chat/completions` | Bedrock Invoke | `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py` |
+| `/v1/chat/completions` | Gemini | `llms/gemini/chat/transformation.py` |
+| `/v1/chat/completions` | Vertex AI | `llms/vertex_ai/gemini/transformation.py` |
+| `/v1/chat/completions` | OpenAI | `llms/openai/chat/gpt_transformation.py` |
+| `/v1/messages` (passthrough) | Anthropic | `llms/anthropic/experimental_pass_through/messages/transformation.py` |
+| `/v1/messages` (passthrough) | Bedrock | `llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py` |
+| `/v1/messages` (passthrough) | Vertex AI | `llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py` |
+| Passthrough endpoints | All | `proxy/pass_through_endpoints/llm_provider_handlers/` |
+
+### Example: Debugging prompt caching
+
+If `/v1/messages` → Bedrock Converse prompt caching isn't working but Bedrock Invoke works:
+
+1. **Bedrock Converse translation**: `llms/bedrock/chat/converse_transformation.py`
+2. **Bedrock Invoke translation**: `llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py`
+3. Compare how each handles `cache_control` in `transform_request()`
+
+### How translations work
+
+Each provider has a `Config` class that inherits from `BaseConfig` (`llms/base_llm/chat/transformation.py`):
+
+```python
+class ProviderConfig(BaseConfig):
+    def transform_request(self, model, messages, optional_params, litellm_params, headers):
+        # Convert OpenAI format → Provider format
+        return {"messages": transformed_messages, ...}
+    
+    def transform_response(self, model, raw_response, model_response, logging_obj, ...):
+        # Convert Provider format → OpenAI format
+        return ModelResponse(choices=[...], usage=Usage(...))
+```
+
+The `BaseLLMHTTPHandler` (`llms/custom_httpx/llm_http_handler.py`) calls these methods - you never need to modify the handler itself.
+
+---
+
+## 4. Adding/Modifying Providers
+
+### To add a new provider:
+
+1. Create `llms/{provider}/chat/transformation.py`
+2. Implement `Config` class with `transform_request()` and `transform_response()`
+3. Add tests in `tests/llm_translation/test_{provider}.py`
+
+### To add a feature (e.g., prompt caching):
+
+1. Find the translation file from the table above
+2. Modify `transform_request()` to handle the new parameter
+3. Add unit tests that verify the transformation
+
+### Testing checklist
+
+When adding a feature, verify it works across all paths:
+
+| Test | File Pattern |
+|------|--------------|
+| OpenAI passthrough | `tests/llm_translation/test_openai*.py` |
+| Anthropic direct | `tests/llm_translation/test_anthropic*.py` |
+| Bedrock Invoke | `tests/llm_translation/test_bedrock*.py` |
+| Bedrock Converse | `tests/llm_translation/test_bedrock*converse*.py` |
+| Vertex AI | `tests/llm_translation/test_vertex*.py` |
+| Gemini | `tests/llm_translation/test_gemini*.py` |
+
+### Unit testing translations
+
+Translations are designed to be unit testable without making API calls:
+
+```python
+from litellm.llms.bedrock.chat.converse_transformation import BedrockConverseConfig
+
+def test_prompt_caching_transform():
+    config = BedrockConverseConfig()
+    result = config.transform_request(
+        model="anthropic.claude-3-opus",
+        messages=[{"role": "user", "content": "test", "cache_control": {"type": "ephemeral"}}],
+        optional_params={},
+        litellm_params={},
+        headers={}
+    )
+    assert "cachePoint" in str(result)  # Verify cache_control was translated
+```

CLAUDE.md

@@ -25,6 +25,25 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 - `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
 - `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
 
+### Running Scripts
+- `poetry run python script.py` - Run Python scripts (use for non-test files)
+
+### GitHub Issue & PR Templates
+When contributing to the project, use the appropriate templates:
+
+**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
+- Describe what happened vs. what you expected
+- Include relevant log output
+- Specify your LiteLLM version
+
+**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
+- Describe the feature clearly
+- Explain the motivation and use case
+
+**Pull Requests** (`.github/pull_request_template.md`):
+- Add at least 1 test in `tests/litellm/`
+- Ensure `make test-unit` passes
+
 ## Architecture Overview
 
 LiteLLM is a unified interface for 100+ LLM providers with two main components:
@@ -71,19 +90,70 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - Pydantic v2 for data validation
 - Async/await patterns throughout
 - Type hints required for all public APIs
+- **Avoid imports within methods** — place all imports at the top of the file (module-level). Inline imports inside functions/methods make dependencies harder to trace and hurt readability. The only exception is avoiding circular imports where absolutely necessary.
+- **Use dict spread for immutable copies** — prefer `{**original, "key": new_value}` over `dict(obj)` + mutation. The spread produces the final dict in one step and makes intent clear.
+- **Guard at resolution time** — when resolving an optional value through a fallback chain (`a or b or ""`), raise immediately if the resolved result being empty is an error. Don't pass empty strings or sentinel values downstream for the callee to deal with.
+- **Extract complex comprehensions to named helpers** — a set/dict comprehension that calls into the DB or manager (e.g. "which of these server IDs are OAuth2?") belongs in a named helper function, not inline in the caller.
+- **FastAPI parameter declarations** — mark required query/form params with `= Query(...)` / `= Form(...)` explicitly when other params in the same handler are optional. Mixing `str` (required) with `Optional[str] = None` in the same signature causes silent 422s when the required param is missing.
 
 ### Testing Strategy
 - Unit tests in `tests/test_litellm/`
 - Integration tests for each provider in `tests/llm_translation/`
 - Proxy tests in `tests/proxy_unit_tests/`
 - Load tests in `tests/load_tests/`
+- **Always add tests when adding new entity types or features** — if the existing test file covers other entity types, add corresponding tests for the new one
+- **Keep monkeypatch stubs in sync with real signatures** — when a function gains a new optional parameter, update every `fake_*` / `stub_*` in tests that patch it to also accept that kwarg (even as `**kwargs`). Stale stubs fail with `unexpected keyword argument` and mask real bugs.
+- **Test all branches of name→ID resolution** — when adding server/resource lookup that resolves names to UUIDs, test: (1) name resolves and UUID is allowed, (2) name resolves but UUID is not allowed, (3) name does not resolve at all. The silent-fallback path is where access-control bugs hide.
+
+### UI / Backend Consistency
+- When wiring a new UI entity type to an existing backend endpoint, verify the backend API contract (single value vs. array, required vs. optional params) and ensure the UI controls match — e.g., use a single-select dropdown when the backend accepts a single value, not a multi-select
+
+### MCP OAuth / OpenAPI Transport Mapping
+- `TRANSPORT.OPENAPI` is a UI-only concept. The backend only accepts `"http"`, `"sse"`, or `"stdio"`. Always map it to `"http"` before any API call (including pre-OAuth temp-session calls).
+- FastAPI validation errors return `detail` as an array of `{loc, msg, type}` objects. Error extractors must handle: array (map `.msg`), string, nested `{error: string}`, and fallback.
+- When an MCP server already has `authorization_url` stored, skip OAuth discovery (`_discovery_metadata`) — the server URL for OpenAPI MCPs is the spec file, not the API base, and fetching it causes timeouts.
+- `client_id` should be optional in the `/authorize` endpoint — if the server has a stored `client_id` in credentials, use that. Never require callers to re-supply it.
+
+### MCP Credential Storage
+- OAuth credentials and BYOK credentials share the `litellm_mcpusercredentials` table, distinguished by a `"type"` field in the JSON payload (`"oauth2"` vs plain string).
+- When deleting OAuth credentials, check type before deleting to avoid accidentally deleting a BYOK credential for the same `(user_id, server_id)` pair.
+- Always pass the raw `expires_at` timestamp to the client — never set it to `None` for expired credentials. Let the frontend compute the "Expired" display state from the timestamp.
+- Use `RecordNotFoundError` (not bare `except Exception`) when catching "already deleted" in credential delete endpoints.
+
+### Browser Storage Safety (UI)
+- Never write LiteLLM access tokens or API keys to `localStorage` — use `sessionStorage` only. `localStorage` survives browser close and is readable by any injected script (XSS).
+- Shared utility functions (e.g. `extractErrorMessage`) belong in `src/utils/` — never define them inline in hooks or duplicate them across files.
 
 ### Database Migrations
 - Prisma handles schema migrations
 - Migration files auto-generated with `prisma migrate dev`
 - Always test migrations against both PostgreSQL and SQLite
 
+### Proxy database access
+- **Do not write raw SQL** for proxy DB operations. Use Prisma model methods instead of `execute_raw` / `query_raw`.
+- Use the generated client: `prisma_client.db.<model>` (e.g. `litellm_tooltable`, `litellm_usertable`) with `.upsert()`, `.find_many()`, `.find_unique()`, `.update()`, `.update_many()` as appropriate. This avoids schema/client drift, keeps code testable with simple mocks, and matches patterns used in spend logs and other proxy code.
+- **No N+1 queries.** Never query the DB inside a loop. Batch-fetch with `{"in": ids}` and distribute in-memory.
+- **Batch writes.** Use `create_many`/`update_many`/`delete_many` instead of individual calls (these return counts only; `update_many`/`delete_many` no-op silently on missing rows). When multiple separate writes target the same table (e.g. in `batch_()`), order by primary key to avoid deadlocks.
+- **Push work to the DB.** Filter, sort, group, and aggregate in SQL, not Python. Verify Prisma generates the expected SQL — e.g. prefer `group_by` over `find_many(distinct=...)` which does client-side processing.
+- **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
+- **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
+- **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
+- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
+
 ### Enterprise Features
 - Enterprise-specific code in `enterprise/` directory
 - Optional features enabled via environment variables
-- Separate licensing and authentication for enterprise features
+- Separate licensing and authentication for enterprise features
+
+### HTTP Client Cache Safety
+- **Never close HTTP/SDK clients on cache eviction.** `LLMClientCache._remove_key()` must not call `close()`/`aclose()` on evicted clients — they may still be used by in-flight requests. Doing so causes `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expires. Cleanup happens at shutdown via `close_litellm_async_clients()`.
+
+### Troubleshooting: DB schema out of sync after proxy restart
+`litellm-proxy-extras` runs `prisma migrate deploy` on startup using **its own** bundled migration files, which may lag behind schema changes in the current worktree. Symptoms: `Unknown column`, `Invalid prisma invocation`, or missing data on new fields.
+
+**Diagnose:** Run `\d "TableName"` in psql and compare against `schema.prisma` — missing columns confirm the issue.
+
+**Fix options:**
+1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name <description>` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup.
+2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production.
+3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it.

CONTRIBUTING.md

@@ -7,11 +7,20 @@ Thank you for your interest in contributing to LiteLLM! We welcome contributions
 Here are the core requirements for any PR submitted to LiteLLM:
 
 - [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
+- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+
+#### Proxy (Backend) PRs
+
 - [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
 - [ ] **Ensure your PR passes all checks**:
   - [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
   - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
-- [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+
+#### UI PRs
+
+- [ ] **Ensure the UI builds successfully** - `npm run build`
+- [ ] **Ensure all UI unit tests pass** - `npm run test`
+- [ ] **Add tests for new components or logic** - If you are adding a new component or new logic, add corresponding tests
 
 ## **Contributor License Agreement (CLA)**
 
@@ -24,8 +33,9 @@ Before contributing code to LiteLLM, you must sign our [Contributor License Agre
 ### 1. Setup Your Local Development Environment
 
 ```bash
-# Clone the repository
-git clone https://github.com/BerriAI/litellm.git
+# Fork the repository on GitHub (click the Fork button at https://github.com/BerriAI/litellm)
+# Then clone your fork locally
+git clone https://github.com/YOUR_USERNAME/litellm.git
 cd litellm
 
 # Create a new branch for your feature
@@ -244,6 +254,43 @@ docker run \
     --config /app/config.yaml --detailed_debug
 ```
 
+## UI Development
+
+### 1. Setup Your Local UI Development Environment
+
+```bash
+# Clone the repo (if you haven't already)
+git clone https://github.com/YOUR_USERNAME/litellm.git
+cd litellm
+
+# Navigate to the UI dashboard directory
+cd ui/litellm-dashboard
+
+# Install dependencies
+npm install
+
+# Start the development server
+npm run dev
+```
+
+### 2. Adding UI Tests
+
+If you are adding a **new component** or **new logic**, you must add corresponding tests.
+
+### 3. Running UI Unit Tests
+
+```bash
+npm run test
+```
+
+### 4. Building the UI
+
+Ensure the UI builds successfully before submitting your PR:
+
+```bash
+npm run build
+```
+
 ## Submitting Your PR
 
 1. **Push your branch**: `git push origin your-feature-branch`

Dockerfile

@@ -1,8 +1,9 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base
+
 # Builder stage
 FROM $LITELLM_BUILD_IMAGE AS builder
 
@@ -12,17 +13,16 @@ WORKDIR /app
 USER root
 
 # Install build dependencies
-RUN apk add --no-cache gcc python3-dev openssl openssl-dev
+RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev
 
-
-RUN pip install --upgrade pip>=24.3.1 && \
-    pip install build
+RUN python -m pip install build
 
 # Copy the current directory contents into the container at /app
 COPY . .
 
 # Build Admin UI
-RUN chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
+# Convert Windows line endings to Unix and make executable
+RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
 
 # Build the package
 RUN rm -rf dist/* && python -m build
@@ -47,11 +47,38 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 # Ensure runtime stage runs as root
 USER root
 
-# Install runtime dependencies
-RUN apk add --no-cache openssl tzdata
-
-# Upgrade pip to fix CVE-2025-8869
-RUN pip install --upgrade pip>=24.3.1
+# Install runtime dependencies (libsndfile needed for audio processing on ARM64)
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip libsndfile && \
+    npm install -g npm@latest tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
+    # SECURITY FIX: npm bundles tar, glob, and brace-expansion at multiple nested
+    # levels inside its dependency tree. `npm install -g <pkg>` only creates a
+    # SEPARATE global package, it does NOT replace npm's internal copies.
+    # We must find and replace EVERY copy inside npm's directory.
+    GLOBAL="$(npm root -g)" && \
+    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done && \
+    # SECURITY FIX: patch npm's own package.json metadata so scanners see the
+    # actual installed versions instead of the stale declared dependencies.
+    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
+        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
+    npm cache clean --force && \
+    # Remove the apk-tracked npm so its stale SBOM metadata (tar 7.5.9) is
+    # no longer visible to image scanners.  The globally installed npm@latest
+    # at /usr/local/lib/node_modules/npm/ remains fully functional.
+    { apk del --no-cache npm 2>/dev/null || true; }
 
 WORKDIR /app
 # Copy the current directory contents into the container at /app
@@ -65,17 +92,44 @@ COPY --from=builder /wheels/ /wheels/
 # Install the built wheel using pip; again using a wildcard if it's the only file
 RUN pip install *.whl /wheels/* --no-index --find-links=/wheels/ && rm -f *.whl && rm -rf /wheels
 
+# Replace the nodejs-wheel-binaries bundled node with the system node (fixes CVE-2025-55130)
+RUN NODEJS_WHEEL_NODE=$(find /usr/lib -path "*/nodejs_wheel/bin/node" 2>/dev/null) && \
+    if [ -n "$NODEJS_WHEEL_NODE" ]; then cp /usr/bin/node "$NODEJS_WHEEL_NODE"; fi
+
 # Remove test files and keys from dependencies
 RUN find /usr/lib -type f -path "*/tornado/test/*" -delete && \
     find /usr/lib -type d -path "*/tornado/test" -delete
 
-# Install semantic_router and aurelio-sdk using script
-RUN chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+# SECURITY FIX: nodejs-wheel-binaries (pip package used by Prisma) bundles a complete
+# npm with old vulnerable deps at /usr/lib/python3.*/site-packages/nodejs_wheel/.
+# Patch every copy of tar, glob, and brace-expansion inside that tree.
+RUN GLOBAL="$(npm root -g)" && \
+    [ -n "$GLOBAL" ] || { echo "ERROR: npm root -g returned empty; aborting"; exit 1; } && \
+    find /usr/lib -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
+    done && \
+    find /usr/lib -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
+    done && \
+    find /usr/lib -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
+    done && \
+    find /usr/lib -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
+    done && \
+    find /usr/lib -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
+        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
+    done
 
-# Generate prisma client
-RUN prisma generate
-RUN chmod +x docker/entrypoint.sh
-RUN chmod +x docker/prod_entrypoint.sh
+# Install semantic_router and aurelio-sdk using script
+# Convert Windows line endings to Unix and make executable
+RUN sed -i 's/\r$//' docker/install_auto_router.sh && chmod +x docker/install_auto_router.sh && ./docker/install_auto_router.sh
+
+# Generate prisma client using the correct schema
+RUN prisma generate --schema=./litellm/proxy/schema.prisma
+# Convert Windows line endings to Unix for entrypoint scripts
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
+RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
 
 EXPOSE 4000/tcp
 

GEMINI.md

@@ -25,6 +25,25 @@ This file provides guidance to Gemini when working with code in this repository.
 - `poetry run pytest tests/path/to/test_file.py -v` - Run specific test file
 - `poetry run pytest tests/path/to/test_file.py::test_function -v` - Run specific test
 
+### Running Scripts
+- `poetry run python script.py` - Run Python scripts (use for non-test files)
+
+### GitHub Issue & PR Templates
+When contributing to the project, use the appropriate templates:
+
+**Bug Reports** (`.github/ISSUE_TEMPLATE/bug_report.yml`):
+- Describe what happened vs. what you expected
+- Include relevant log output
+- Specify your LiteLLM version
+
+**Feature Requests** (`.github/ISSUE_TEMPLATE/feature_request.yml`):
+- Describe the feature clearly
+- Explain the motivation and use case
+
+**Pull Requests** (`.github/pull_request_template.md`):
+- Add at least 1 test in `tests/litellm/`
+- Ensure `make test-unit` passes
+
 ## Architecture Overview
 
 LiteLLM is a unified interface for 100+ LLM providers with two main components:

Makefile

@@ -1,7 +1,12 @@
 # LiteLLM Makefile
 # Simple Makefile for running tests and basic development tasks
 
-.PHONY: help test test-unit test-integration test-unit-helm lint format install-dev install-proxy-dev install-test-deps install-helm-unittest check-circular-imports check-import-safety
+.PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
+	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
+	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
+	info lint lint-dev format \
+	install-dev install-proxy-dev install-test-deps \
+	install-helm-unittest check-circular-imports check-import-safety
 
 # Default target
 help:
@@ -22,9 +27,26 @@ help:
 	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
 	@echo "  make test-unit          - Run unit tests (tests/test_litellm)"
+	@echo "  make test-unit-llms     - Run LLM provider tests (~225 files)"
+	@echo "  make test-unit-proxy-guardrails - Run proxy guardrails+mgmt tests (~51 files)"
+	@echo "  make test-unit-proxy-core - Run proxy auth+client+db+hooks tests (~52 files)"
+	@echo "  make test-unit-proxy-misc - Run proxy misc tests (~77 files)"
+	@echo "  make test-unit-integrations - Run integration tests (~60 files)"
+	@echo "  make test-unit-core-utils - Run core utils tests (~32 files)"
+	@echo "  make test-unit-other    - Run other tests (caching, responses, etc., ~69 files)"
+	@echo "  make test-unit-root     - Run root-level tests (~34 files)"
+	@echo "  make test-proxy-unit-a  - Run proxy_unit_tests (a-o, ~20 files)"
+	@echo "  make test-proxy-unit-b  - Run proxy_unit_tests (p-z, ~28 files)"
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
 
+# Keep PIP simple for edge cases:
+PIP := $(shell command -v pip > /dev/null 2>&1 && echo "pip" || echo "python3 -m pip")
+
+# Show info
+info:
+	@echo "PIP: $(PIP)"
+
 # Installation targets
 install-dev:
 	poetry install --with dev
@@ -34,18 +56,19 @@ install-proxy-dev:
 
 # CI-compatible installations (matches GitHub workflows exactly)
 install-dev-ci:
-	pip install openai==1.99.5
+	$(PIP) install openai==2.8.0
 	poetry install --with dev
-	pip install openai==1.99.5
+	$(PIP) install openai==2.8.0
 
 install-proxy-dev-ci:
 	poetry install --with dev,proxy-dev --extras proxy
-	pip install openai==1.99.5
+	$(PIP) install openai==2.8.0
 
 install-test-deps: install-proxy-dev
-	poetry run pip install "pytest-retry==1.6.3"
-	poetry run pip install pytest-xdist
-	cd enterprise && poetry run pip install -e . && cd ..
+	poetry run $(PIP) install "pytest-retry==1.6.3"
+	poetry run $(PIP) install pytest-xdist
+	poetry run $(PIP) install openapi-core
+	cd enterprise && poetry run $(PIP) install -e . && cd ..
 
 install-helm-unittest:
 	helm plugin install https://github.com/helm-unittest/helm-unittest --version v0.4.4 || echo "ignore error if plugin exists"
@@ -61,8 +84,40 @@ format-check: install-dev
 lint-ruff: install-dev
 	cd litellm && poetry run ruff check . && cd ..
 
+# faster linter for developing ...
+# inspiration from:
+# https://github.com/astral-sh/ruff/discussions/10977
+# https://github.com/astral-sh/ruff/discussions/4049
+lint-format-changed: install-dev
+	@git diff origin/main --unified=0 --no-color -- '*.py' | \
+	perl -ne '\
+		if (/^diff --git a\/(.*) b\//) { $$file = $$1; } \
+		if (/^@@ .* \+(\d+)(?:,(\d+))? @@/) { \
+			$$start = $$1; $$count = $$2 || 1; $$end = $$start + $$count - 1; \
+			print "$$file:$$start:1-$$end:999\n"; \
+		}' | \
+	while read range; do \
+		file="$${range%%:*}"; \
+		lines="$${range#*:}"; \
+		echo "Formatting $$file (lines $$lines)"; \
+		poetry run ruff format --range "$$lines" "$$file"; \
+	done
+
+lint-ruff-dev: install-dev
+	@tmpfile=$$(mktemp /tmp/ruff-dev.XXXXXX) && \
+	cd litellm && \
+	(poetry run ruff check . --output-format=pylint || true) > "$$tmpfile" && \
+	poetry run diff-quality --violations=pylint "$$tmpfile" --compare-branch=origin/main && \
+	cd .. ; \
+	rm -f "$$tmpfile"
+
+lint-ruff-FULL-dev: install-dev
+	@files=$$(git diff --name-only origin/main -- '*.py'); \
+	if [ -n "$$files" ]; then echo "$$files" | xargs poetry run ruff check; \
+	else echo "No changed .py files to check."; fi
+
 lint-mypy: install-dev
-	poetry run pip install types-requests types-setuptools types-redis types-PyYAML
+	poetry run $(PIP) install types-requests types-setuptools types-redis types-PyYAML
 	cd litellm && poetry run mypy . --ignore-missing-imports && cd ..
 
 lint-black: format-check
@@ -71,11 +126,14 @@ check-circular-imports: install-dev
 	cd litellm && poetry run python ../tests/documentation_tests/test_circular_imports.py && cd ..
 
 check-import-safety: install-dev
-	poetry run python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
+	@poetry run python -c "from litellm import *; print('[from litellm import *] OK! no issues!');" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
 
 # Combined linting (matches test-linting.yml workflow)
 lint: format-check lint-ruff lint-mypy check-circular-imports check-import-safety
 
+# Faster linting for local development (only checks changed code)
+lint-dev: lint-format-changed lint-mypy check-circular-imports check-import-safety
+
 # Testing targets
 test:
 	poetry run pytest tests/
@@ -83,6 +141,38 @@ test:
 test-unit: install-test-deps
 	poetry run pytest tests/test_litellm -x -vv -n 4
 
+# Matrix test targets (matching CI workflow groups)
+test-unit-llms: install-test-deps
+	poetry run pytest tests/test_litellm/llms --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-guardrails: install-test-deps
+	poetry run pytest tests/test_litellm/proxy/guardrails tests/test_litellm/proxy/management_endpoints tests/test_litellm/proxy/management_helpers --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-core: install-test-deps
+	poetry run pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20
+
+test-unit-proxy-misc: install-test-deps
+	poetry run pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20
+
+test-unit-integrations: install-test-deps
+	poetry run pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20
+
+test-unit-core-utils: install-test-deps
+	poetry run pytest tests/test_litellm/litellm_core_utils --tb=short -vv -n 2 --durations=20
+
+test-unit-other: install-test-deps
+	poetry run pytest tests/test_litellm/caching tests/test_litellm/responses tests/test_litellm/secret_managers tests/test_litellm/vector_stores tests/test_litellm/a2a_protocol tests/test_litellm/anthropic_interface tests/test_litellm/completion_extras tests/test_litellm/containers tests/test_litellm/enterprise tests/test_litellm/experimental_mcp_client tests/test_litellm/google_genai tests/test_litellm/images tests/test_litellm/interactions tests/test_litellm/passthrough tests/test_litellm/router_strategy tests/test_litellm/router_utils tests/test_litellm/types --tb=short -vv -n 4 --durations=20
+
+test-unit-root: install-test-deps
+	poetry run pytest tests/test_litellm/test_*.py --tb=short -vv -n 4 --durations=20
+
+# Proxy unit tests (tests/proxy_unit_tests split alphabetically)
+test-proxy-unit-a: install-test-deps
+	poetry run pytest tests/proxy_unit_tests/test_[a-o]*.py --tb=short -vv -n 2 --durations=20
+
+test-proxy-unit-b: install-test-deps
+	poetry run pytest tests/proxy_unit_tests/test_[p-z]*.py --tb=short -vv -n 2 --durations=20
+
 test-integration:
 	poetry run pytest tests/ -k "not test_litellm"
 
@@ -100,4 +190,4 @@ test-llm-translation-single: install-test-deps
 	@mkdir -p test-results
 	poetry run pytest tests/llm_translation/$(FILE) \
 		--junitxml=test-results/junit.xml \
-		-v --tb=short --maxfail=100 --timeout=300
+		-v --tb=short --maxfail=100 --timeout=300

README.md

@@ -2,16 +2,16 @@
         🚅 LiteLLM
     </h1>
     <p align="center">
+        <p align="center">Call 100+ LLMs in OpenAI format. [Bedrock, Azure, OpenAI, VertexAI, Anthropic, Groq, etc.]
+        </p>
         <p align="center">
         <a href="https://render.com/deploy?repo=https://github.com/BerriAI/litellm" target="_blank" rel="nofollow"><img src="https://render.com/images/deploy-to-render-button.svg" alt="Deploy to Render"></a>
         <a href="https://railway.app/template/HLP0Ub?referralCode=jch2ME">
           <img src="https://railway.app/button.svg" alt="Deploy on Railway">
         </a>
         </p>
-        <p align="center">Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]
-        <br>
     </p>
-<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (LLM Gateway)</a> | <a href="https://docs.litellm.ai/docs/hosted" target="_blank"> Hosted Proxy (Preview)</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
+<h4 align="center"><a href="https://docs.litellm.ai/docs/simple_proxy" target="_blank">LiteLLM Proxy Server (AI Gateway)</a> | <a href="https://docs.litellm.ai/docs/enterprise#hosted-litellm-proxy" target="_blank"> Hosted Proxy</a> | <a href="https://docs.litellm.ai/docs/enterprise"target="_blank">Enterprise Tier</a></h4>
 <h4 align="center">
     <a href="https://pypi.org/project/litellm/" target="_blank">
         <img src="https://img.shields.io/pypi/v/litellm.svg" alt="PyPI Version">
@@ -30,31 +30,17 @@
     </a>
 </h4>
 
-LiteLLM manages:
+<img width="2688" height="1600" alt="Group 7154 (1)" src="https://github.com/user-attachments/assets/c5ee0412-6fb5-4fb6-ab5b-bafae4209ca6" />
 
-- Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints
-- [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']`
-- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing)
-- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server (LLM Gateway)](https://docs.litellm.ai/docs/simple_proxy)
 
-LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
+## Use LiteLLM for
 
-[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#litellm-proxy-server-llm-gateway---docs) <br>
-[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs)
+<details open>
+<summary><b>LLMs</b> - Call 100+ LLMs (Python SDK + AI Gateway)</summary>
 
-🚨 **Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
+[**All Supported Endpoints**](https://docs.litellm.ai/docs/supported_endpoints) - `/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, `/rerank`, `/a2a`, `/messages` and more.
 
-Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
-
-# Usage ([**Docs**](https://docs.litellm.ai/docs/))
-
-> [!IMPORTANT]
-> LiteLLM v1.0.0 now requires `openai>=1.0.0`. Migration guide [here](https://docs.litellm.ai/docs/migration)
-> LiteLLM v1.40.14+ now requires `pydantic>=2.0.0`. No changes required.
-
-<a target="_blank" href="https://colab.research.google.com/github/BerriAI/litellm/blob/main/cookbook/liteLLM_Getting_Started.ipynb">
-  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
-</a>
+### Python SDK
 
 ```shell
 pip install litellm
@@ -64,257 +50,237 @@ pip install litellm
 from litellm import completion
 import os
 
-## set ENV variables
 os.environ["OPENAI_API_KEY"] = "your-openai-key"
 os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key"
 
-messages = [{ "content": "Hello, how are you?","role": "user"}]
+# OpenAI
+response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hello!"}])
 
-# openai call
-response = completion(model="openai/gpt-4o", messages=messages)
-
-# anthropic call
-response = completion(model="anthropic/claude-sonnet-4-20250514", messages=messages)
-print(response)
+# Anthropic  
+response = completion(model="anthropic/claude-sonnet-4-20250514", messages=[{"role": "user", "content": "Hello!"}])
 ```
 
-### Response (OpenAI Format)
+### AI Gateway (Proxy Server)
 
-```json
-{
-    "id": "chatcmpl-1214900a-6cdd-4148-b663-b5e2f642b4de",
-    "created": 1751494488,
-    "model": "claude-sonnet-4-20250514",
-    "object": "chat.completion",
-    "system_fingerprint": null,
-    "choices": [
-        {
-            "finish_reason": "stop",
-            "index": 0,
-            "message": {
-                "content": "Hello! I'm doing well, thank you for asking. I'm here and ready to help with whatever you'd like to discuss or work on. How are you doing today?",
-                "role": "assistant",
-                "tool_calls": null,
-                "function_call": null
-            }
-        }
-    ],
-    "usage": {
-        "completion_tokens": 39,
-        "prompt_tokens": 13,
-        "total_tokens": 52,
-        "completion_tokens_details": null,
-        "prompt_tokens_details": {
-            "audio_tokens": null,
-            "cached_tokens": 0
-        },
-        "cache_creation_input_tokens": 0,
-        "cache_read_input_tokens": 0
-    }
-}
-```
-
-Call any model supported by a provider, with `model=<provider_name>/<model_name>`. There might be provider-specific details here, so refer to [provider docs for more information](https://docs.litellm.ai/docs/providers)
-
-## Async ([Docs](https://docs.litellm.ai/docs/completion/stream#async-completion))
-
-```python
-from litellm import acompletion
-import asyncio
-
-async def test_get_response():
-    user_message = "Hello, how are you?"
-    messages = [{"content": user_message, "role": "user"}]
-    response = await acompletion(model="openai/gpt-4o", messages=messages)
-    return response
-
-response = asyncio.run(test_get_response())
-print(response)
-```
-
-## Streaming ([Docs](https://docs.litellm.ai/docs/completion/stream))
-
-LiteLLM supports streaming the model response back, pass `stream=True` to get a streaming iterator in response.
-Streaming is supported for all models (Bedrock, Huggingface, TogetherAI, Azure, OpenAI, etc.)
-
-```python
-from litellm import completion
-
-messages = [{"content": "Hello, how are you?", "role": "user"}]
-
-# gpt-4o
-response = completion(model="openai/gpt-4o", messages=messages, stream=True)
-for part in response:
-    print(part.choices[0].delta.content or "")
-
-# claude sonnet 4
-response = completion('anthropic/claude-sonnet-4-20250514', messages, stream=True)
-for part in response:
-    print(part)
-```
-
-### Response chunk (OpenAI Format)
-
-```json
-{
-    "id": "chatcmpl-fe575c37-5004-4926-ae5e-bfbc31f356ca",
-    "created": 1751494808,
-    "model": "claude-sonnet-4-20250514",
-    "object": "chat.completion.chunk",
-    "system_fingerprint": null,
-    "choices": [
-        {
-            "finish_reason": null,
-            "index": 0,
-            "delta": {
-                "provider_specific_fields": null,
-                "content": "Hello",
-                "role": "assistant",
-                "function_call": null,
-                "tool_calls": null,
-                "audio": null
-            },
-            "logprobs": null
-        }
-    ],
-    "provider_specific_fields": null,
-    "stream_options": null,
-    "citations": null
-}
-```
-
-## Logging Observability ([Docs](https://docs.litellm.ai/docs/observability/callbacks))
-
-LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, DynamoDB, s3 Buckets, Helicone, Promptlayer, Traceloop, Athina, Slack
-
-```python
-from litellm import completion
-
-## set env variables for logging tools (when using MLflow, no API key set up is required)
-os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key"
-os.environ["HELICONE_API_KEY"] = "your-helicone-auth-key"
-os.environ["LANGFUSE_PUBLIC_KEY"] = ""
-os.environ["LANGFUSE_SECRET_KEY"] = ""
-os.environ["ATHINA_API_KEY"] = "your-athina-api-key"
-
-os.environ["OPENAI_API_KEY"] = "your-openai-key"
-
-# set callbacks
-litellm.success_callback = ["lunary", "mlflow", "langfuse", "athina", "helicone"] # log input/output to lunary, langfuse, supabase, athina, helicone etc
-
-#openai call
-response = completion(model="openai/gpt-4o", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}])
-```
-
-# LiteLLM Proxy Server (LLM Gateway) - ([Docs](https://docs.litellm.ai/docs/simple_proxy))
-
-Track spend + Load Balance across multiple projects
-
-[Hosted Proxy (Preview)](https://docs.litellm.ai/docs/hosted)
-
-The proxy provides:
-
-1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth)
-2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class)
-3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend)
-4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits)
-
-## 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/)
-
-
-## Quick Start Proxy - CLI
+[**Getting Started - E2E Tutorial**](https://docs.litellm.ai/docs/proxy/docker_quick_start) - Setup virtual keys, make your first request
 
 ```shell
 pip install 'litellm[proxy]'
+litellm --model gpt-4o
 ```
 
-### Step 1: Start litellm proxy
-
-```shell
-$ litellm --model huggingface/bigcode/starcoder
-
-#INFO: Proxy running on http://0.0.0.0:4000
-```
-
-### Step 2: Make ChatCompletions Request to Proxy
-
-
-> [!IMPORTANT]
-> 💡 [Use LiteLLM Proxy with Langchain (Python, JS), OpenAI SDK (Python, JS) Anthropic SDK, Mistral SDK, LlamaIndex, Instructor, Curl](https://docs.litellm.ai/docs/proxy/user_keys)
-
 ```python
-import openai # openai v1.0.0+
-client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url
-# request sent to model set on litellm proxy, `litellm --model`
-response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
-    {
-        "role": "user",
-        "content": "this is a test request, write a short poem"
-    }
-])
+import openai
 
-print(response)
+client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000")
+response = client.chat.completions.create(
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Hello!"}]
+)
 ```
 
-## Proxy Key Management ([Docs](https://docs.litellm.ai/docs/proxy/virtual_keys))
+[**Docs: LLM Providers**](https://docs.litellm.ai/docs/providers)
 
-Connect the proxy with a Postgres DB to create proxy keys
+</details>
+
+<details>
+<summary><b>Agents</b> - Invoke A2A Agents (Python SDK + AI Gateway)</summary>
+
+[**Supported Providers**](https://docs.litellm.ai/docs/a2a#add-a2a-agents) - LangGraph, Vertex AI Agent Engine, Azure AI Foundry, Bedrock AgentCore, Pydantic AI
+
+### Python SDK - A2A Protocol
+
+```python
+from litellm.a2a_protocol import A2AClient
+from a2a.types import SendMessageRequest, MessageSendParams
+from uuid import uuid4
+
+client = A2AClient(base_url="http://localhost:10001")
+
+request = SendMessageRequest(
+    id=str(uuid4()),
+    params=MessageSendParams(
+        message={
+            "role": "user",
+            "parts": [{"kind": "text", "text": "Hello!"}],
+            "messageId": uuid4().hex,
+        }
+    )
+)
+response = await client.send_message(request)
+```
+
+### AI Gateway (Proxy Server)
+
+**Step 1.** [Add your Agent to the AI Gateway](https://docs.litellm.ai/docs/a2a#adding-your-agent)
+
+**Step 2.** Call Agent via A2A SDK
+
+```python
+from a2a.client import A2ACardResolver, A2AClient
+from a2a.types import MessageSendParams, SendMessageRequest
+from uuid import uuid4
+import httpx
+
+base_url = "http://localhost:4000/a2a/my-agent"  # LiteLLM proxy + agent name
+headers = {"Authorization": "Bearer sk-1234"}    # LiteLLM Virtual Key
+
+async with httpx.AsyncClient(headers=headers) as httpx_client:
+    resolver = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
+    agent_card = await resolver.get_agent_card()
+    client = A2AClient(httpx_client=httpx_client, agent_card=agent_card)
+
+    request = SendMessageRequest(
+        id=str(uuid4()),
+        params=MessageSendParams(
+            message={
+                "role": "user",
+                "parts": [{"kind": "text", "text": "Hello!"}],
+                "messageId": uuid4().hex,
+            }
+        )
+    )
+    response = await client.send_message(request)
+```
+
+[**Docs: A2A Agent Gateway**](https://docs.litellm.ai/docs/a2a)
+
+</details>
+
+<details>
+<summary><b>MCP Tools</b> - Connect MCP servers to any LLM (Python SDK + AI Gateway)</summary>
+
+### Python SDK - MCP Bridge
+
+```python
+from mcp import ClientSession, StdioServerParameters
+from mcp.client.stdio import stdio_client
+from litellm import experimental_mcp_client
+import litellm
+
+server_params = StdioServerParameters(command="python", args=["mcp_server.py"])
+
+async with stdio_client(server_params) as (read, write):
+    async with ClientSession(read, write) as session:
+        await session.initialize()
+
+        # Load MCP tools in OpenAI format
+        tools = await experimental_mcp_client.load_mcp_tools(session=session, format="openai")
+
+        # Use with any LiteLLM model
+        response = await litellm.acompletion(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": "What's 3 + 5?"}],
+            tools=tools
+        )
+```
+
+### AI Gateway - MCP Gateway
+
+**Step 1.** [Add your MCP Server to the AI Gateway](https://docs.litellm.ai/docs/mcp#adding-your-mcp)
+
+**Step 2.** Call MCP tools via `/chat/completions`
 
 ```bash
-# Get the code
-git clone https://github.com/BerriAI/litellm
-
-# Go to folder
-cd litellm
-
-# Add the master key - you can change this after setup
-echo 'LITELLM_MASTER_KEY="sk-1234"' > .env
-
-# Add the litellm salt key - you cannot change this after adding a model
-# It is used to encrypt / decrypt your LLM API Key credentials
-# We recommend - https://1password.com/password-generator/
-# password generator to get a random hash for litellm salt key
-echo 'LITELLM_SALT_KEY="sk-1234"' >> .env
-
-source .env
-
-# Start
-docker compose up
+curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
+  -H 'Authorization: Bearer sk-1234' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "model": "gpt-4o",
+    "messages": [{"role": "user", "content": "Summarize the latest open PR"}],
+    "tools": [{
+      "type": "mcp",
+      "server_url": "litellm_proxy/mcp/github",
+      "server_label": "github_mcp",
+      "require_approval": "never"
+    }]
+  }'
 ```
 
+### Use with Cursor IDE
 
-UI on `/ui` on your proxy server
-![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033)
-
-Set budgets and rate limits across multiple projects
-`POST /key/generate`
-
-### Request
-
-```shell
-curl 'http://0.0.0.0:4000/key/generate' \
---header 'Authorization: Bearer sk-1234' \
---header 'Content-Type: application/json' \
---data-raw '{"models": ["gpt-3.5-turbo", "gpt-4", "claude-2"], "duration": "20m","metadata": {"user": "ishaan@berri.ai", "team": "core-infra"}}'
-```
-
-### Expected Response
-
-```shell
+```json
 {
-    "key": "sk-kdEXbIqZRwEeEiHwdg7sFA", # Bearer token
-    "expires": "2023-11-19T01:38:25.838000+00:00" # datetime object
+  "mcpServers": {
+    "LiteLLM": {
+      "url": "http://localhost:4000/mcp/",
+      "headers": {
+        "x-litellm-api-key": "Bearer sk-1234"
+      }
+    }
+  }
 }
 ```
 
+[**Docs: MCP Gateway**](https://docs.litellm.ai/docs/mcp)
+
+</details>
+
+---
+
+## How to use LiteLLM
+
+You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs:
+
+<table style={{width: '100%', tableLayout: 'fixed'}}>
+<thead>
+<tr>
+<th style={{width: '14%'}}></th>
+<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/simple_proxy">LiteLLM AI Gateway</a></strong></th>
+<th style={{width: '43%'}}><strong><a href="https://docs.litellm.ai/docs/">LiteLLM Python SDK</a></strong></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td style={{width: '14%'}}><strong>Use Case</strong></td>
+<td style={{width: '43%'}}>Central service (LLM Gateway) to access multiple LLMs</td>
+<td style={{width: '43%'}}>Use LiteLLM directly in your Python code</td>
+</tr>
+<tr>
+<td style={{width: '14%'}}><strong>Who Uses It?</strong></td>
+<td style={{width: '43%'}}>Gen AI Enablement / ML Platform Teams</td>
+<td style={{width: '43%'}}>Developers building LLM projects</td>
+</tr>
+<tr>
+<td style={{width: '14%'}}><strong>Key Features</strong></td>
+<td style={{width: '43%'}}>Centralized API gateway with authentication and authorization, multi-tenant cost tracking and spend management per project/user, per-project customization (logging, guardrails, caching), virtual keys for secure access control, admin dashboard UI for monitoring and management</td>
+<td style={{width: '43%'}}>Direct Python library integration in your codebase, Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - <a href="https://docs.litellm.ai/docs/routing">Router</a>, application-level load balancing and cost tracking, exception handling with OpenAI-compatible errors, observability callbacks (Lunary, MLflow, Langfuse, etc.)</td>
+</tr>
+</tbody>
+</table>
+
+LiteLLM Performance: **8ms P95 latency** at 1k RPS (See benchmarks [here](https://docs.litellm.ai/docs/benchmarks))
+
+[**Jump to LiteLLM Proxy (LLM Gateway) Docs**](https://docs.litellm.ai/docs/simple_proxy) <br>
+[**Jump to Supported LLM Providers**](https://docs.litellm.ai/docs/providers)
+
+**Stable Release:** Use docker images with the `-stable` tag. These have undergone 12 hour load tests, before being published. [More information about the release cycle here](https://docs.litellm.ai/docs/proxy/release_cycle)
+
+Support for more providers. Missing a provider or LLM Platform, raise a [feature request](https://github.com/BerriAI/litellm/issues/new?assignees=&labels=enhancement&projects=&template=feature_request.yml&title=%5BFeature%5D%3A+).
+
+## OSS Adopters 
+
+<table>
+  <tr>
+    <td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
+    <td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
+    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
+    <td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
+    <td><h2>Netflix</h2></td>
+    <td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
+  </tr>
+</table>
+
 ## Supported Providers ([Website Supported Models](https://models.litellm.ai/) | [Docs](https://docs.litellm.ai/docs/providers))
 
 | Provider                                                                            | `/chat/completions` | `/messages` | `/responses` | `/embeddings` | `/image/generations` | `/audio/transcriptions` | `/audio/speech` | `/moderations` | `/batches` | `/rerank` |
 |-------------------------------------------------------------------------------------|---------------------|-------------|--------------|---------------|----------------------|-------------------------|-----------------|----------------|-----------|-----------|
+| [Abliteration (`abliteration`)](https://docs.litellm.ai/docs/providers/abliteration) | ✅ |  |  |  |  |  |  |  |  |  |
 | [AI/ML API (`aiml`)](https://docs.litellm.ai/docs/providers/aiml) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |
 | [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Amazon Nova](https://docs.litellm.ai/docs/providers/amazon_nova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |  |
 | [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |  |
 | [Anyscale](https://docs.litellm.ai/docs/providers/anyscale) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
@@ -343,14 +309,14 @@ curl 'http://0.0.0.0:4000/key/generate' \
 | [Deepgram (`deepgram`)](https://docs.litellm.ai/docs/providers/deepgram) | ✅ | ✅ | ✅ |  |  | ✅ |  |  |  |  |
 | [DeepInfra (`deepinfra`)](https://docs.litellm.ai/docs/providers/deepinfra) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
-| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  |  | ✅ |  |  |  |
+| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ |  |  |  |
 | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |
 | [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [FriendliAI (`friendliai`)](https://docs.litellm.ai/docs/providers/friendliai) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Galadriel (`galadriel`)](https://docs.litellm.ai/docs/providers/galadriel) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
-| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [GitHub Copilot (`github_copilot`)](https://docs.litellm.ai/docs/providers/github_copilot) | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |  |
 | [GitHub Models (`github`)](https://docs.litellm.ai/docs/providers/github) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Google - PaLM](https://docs.litellm.ai/docs/providers/palm) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Google - Vertex AI (`vertex_ai`)](https://docs.litellm.ai/docs/providers/vertex) | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |  |  |
@@ -421,7 +387,9 @@ curl 'http://0.0.0.0:4000/key/generate' \
 1. (In root) create virtual environment `python -m venv .venv`
 2. Activate virtual environment `source .venv/bin/activate`
 3. Install dependencies `pip install -e ".[all]"`
-4. Start proxy backend `python litellm/proxy_cli.py`
+4. `pip install prisma`
+5. `prisma generate`
+6. Start proxy backend `python litellm/proxy/proxy_cli.py`
 
 ### Frontend
 1. Navigate to `ui/litellm-dashboard`
@@ -431,7 +399,7 @@ curl 'http://0.0.0.0:4000/key/generate' \
 # Enterprise
 For companies that need better security, user management and professional support
 
-[Talk to founders](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+[Talk to founders](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions)
 
 This covers:
 - ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
@@ -503,4 +471,3 @@ All these checks must pass before your PR can be merged.
   <img src="https://contrib.rocks/image?repo=BerriAI/litellm" />
 </a>
 
-

VERTEX_ENV_SETUP.md

@@ -1,261 +0,0 @@
-# Vertex AI Environment Variables Setup Guide
-
-## Overview
-
-LiteLLM can load Vertex AI credentials from environment variables instead of storing them in config files. This is more secure and easier to manage for local development.
-
-## Environment Variables
-
-LiteLLM looks for these environment variables (in order of precedence):
-
-### 1. **DEFAULT_VERTEXAI_PROJECT** (Required)
-Your GCP project ID that has Vertex AI enabled.
-
-```bash
-export DEFAULT_VERTEXAI_PROJECT="my-gcp-project-id"
-```
-
-### 2. **DEFAULT_VERTEXAI_LOCATION** (Required)
-The region/location for Vertex AI services.
-
-```bash
-export DEFAULT_VERTEXAI_LOCATION="global"
-# or
-export DEFAULT_VERTEXAI_LOCATION="us-central1"
-```
-
-Common locations:
-- `global` - For Discovery Engine and global services
-- `us-central1` - US Central region
-- `us-east1` - US East region
-- `europe-west1` - Europe West region
-- `asia-southeast1` - Asia Southeast region
-
-### 3. **DEFAULT_GOOGLE_APPLICATION_CREDENTIALS** (Required)
-Path to your service account JSON key file.
-
-```bash
-export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json"
-```
-
-### 4. **GOOGLE_APPLICATION_CREDENTIALS** (Fallback)
-Standard Google Cloud environment variable (used as fallback).
-
-```bash
-export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account-key.json"
-```
-
-## Quick Setup
-
-### Option 1: Interactive Script
-
-```bash
-chmod +x setup_vertex_env.sh
-source setup_vertex_env.sh
-```
-
-### Option 2: Manual Setup
-
-1. **Set environment variables** (for current session):
-
-```bash
-export DEFAULT_VERTEXAI_PROJECT="your-project-id"
-export DEFAULT_VERTEXAI_LOCATION="global"
-export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"
-export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"
-```
-
-2. **Make them persistent** (add to `~/.zshrc` or `~/.bashrc`):
-
-```bash
-echo 'export DEFAULT_VERTEXAI_PROJECT="your-project-id"' >> ~/.zshrc
-echo 'export DEFAULT_VERTEXAI_LOCATION="global"' >> ~/.zshrc
-echo 'export DEFAULT_GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"' >> ~/.zshrc
-echo 'export GOOGLE_APPLICATION_CREDENTIALS="$HOME/.gcp/service-account.json"' >> ~/.zshrc
-```
-
-3. **Reload your shell**:
-
-```bash
-source ~/.zshrc
-```
-
-## Service Account Setup
-
-### 1. Create a Service Account
-
-```bash
-gcloud iam service-accounts create litellm-vertex-sa \
-    --display-name="LiteLLM Vertex AI Service Account"
-```
-
-### 2. Grant Necessary Permissions
-
-For Discovery Engine (vector stores):
-```bash
-gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \
-    --member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \
-    --role="roles/discoveryengine.viewer"
-
-gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \
-    --member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \
-    --role="roles/discoveryengine.dataStoreEditor"
-```
-
-For general Vertex AI:
-```bash
-gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \
-    --member="serviceAccount:litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com" \
-    --role="roles/aiplatform.user"
-```
-
-### 3. Create and Download Key
-
-```bash
-gcloud iam service-accounts keys create ~/service-account-key.json \
-    --iam-account=litellm-vertex-sa@YOUR_PROJECT_ID.iam.gserviceaccount.com
-```
-
-## Verify Setup
-
-### Check Environment Variables
-
-```bash
-python3 << 'EOF'
-import os
-print("✓ Environment Variables:")
-print(f"  DEFAULT_VERTEXAI_PROJECT: {os.getenv('DEFAULT_VERTEXAI_PROJECT')}")
-print(f"  DEFAULT_VERTEXAI_LOCATION: {os.getenv('DEFAULT_VERTEXAI_LOCATION')}")
-print(f"  DEFAULT_GOOGLE_APPLICATION_CREDENTIALS: {os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')}")
-print(f"  GOOGLE_APPLICATION_CREDENTIALS: {os.getenv('GOOGLE_APPLICATION_CREDENTIALS')}")
-
-# Check if credentials file exists
-creds_path = os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')
-if creds_path and os.path.exists(creds_path):
-    print(f"\n✅ Credentials file found at: {creds_path}")
-else:
-    print(f"\n❌ Credentials file NOT found at: {creds_path}")
-EOF
-```
-
-### Test Authentication
-
-```bash
-python3 << 'EOF'
-import os
-import json
-from google.oauth2 import service_account
-from google.auth.transport.requests import Request
-
-creds_path = os.getenv('DEFAULT_GOOGLE_APPLICATION_CREDENTIALS')
-project = os.getenv('DEFAULT_VERTEXAI_PROJECT')
-
-try:
-    # Load credentials
-    credentials = service_account.Credentials.from_service_account_file(
-        creds_path,
-        scopes=['https://www.googleapis.com/auth/cloud-platform']
-    )
-    
-    # Get access token
-    credentials.refresh(Request())
-    
-    print("✅ Authentication successful!")
-    print(f"   Project: {project}")
-    print(f"   Service Account: {credentials.service_account_email}")
-    print(f"   Token expiry: {credentials.expiry}")
-    
-except Exception as e:
-    print(f"❌ Authentication failed: {e}")
-EOF
-```
-
-## Using with Vector Store Passthrough
-
-Once your environment is set up, the vector store passthrough will work in two ways:
-
-### 1. **With Vector Store Config** (Priority 1)
-If you have a vector store configured with its own credentials in `litellm_params`, those will be used first:
-
-```yaml
-vector_stores:
-  - vector_store_id: test-store-123
-    custom_llm_provider: vertex_ai
-    litellm_params:
-      vertex_project: "specific-project"
-      vertex_location: "us-central1"
-      vertex_credentials: "{...}"  # Inline credentials
-```
-
-### 2. **Environment Variables Fallback** (Priority 2)
-If the vector store doesn't have explicit credentials, it falls back to your environment variables:
-
-```yaml
-vector_stores:
-  - vector_store_id: test-store-123
-    custom_llm_provider: vertex_ai
-    # No litellm_params - will use DEFAULT_VERTEXAI_PROJECT, DEFAULT_VERTEXAI_LOCATION, etc.
-```
-
-### 3. **Model Config Fallback** (Priority 3)
-If neither above work, it looks for credentials in your model configuration.
-
-## Troubleshooting
-
-### "No credentials found"
-
-Check that all environment variables are set:
-```bash
-env | grep -E "(DEFAULT_VERTEXAI|GOOGLE_APPLICATION_CREDENTIALS)"
-```
-
-### "Authentication failed"
-
-Verify your service account key is valid:
-```bash
-cat $DEFAULT_GOOGLE_APPLICATION_CREDENTIALS | python3 -m json.tool
-```
-
-### "Permission denied"
-
-Ensure your service account has the necessary roles:
-```bash
-gcloud projects get-iam-policy YOUR_PROJECT_ID \
-    --flatten="bindings[].members" \
-    --filter="bindings.members:serviceAccount:litellm-vertex-sa@*"
-```
-
-### Different Credentials for Different Projects
-
-If you need to use different credentials for different vector stores, configure them explicitly in the vector store config rather than relying on environment variables.
-
-## Start LiteLLM Proxy
-
-Once your environment is configured:
-
-```bash
-# Start the proxy (it will automatically load env vars)
-litellm --config proxy_server_config.yaml
-
-# Or with debug logging
-export LITELLM_LOG=DEBUG
-litellm --config proxy_server_config.yaml
-```
-
-You should see logs like:
-```
-Vertex: Loading vertex credentials from /path/to/service-account.json
-Found credentials for vertex_ai_default
-```
-
-## Test the Endpoint
-
-```bash
-curl -X POST http://0.0.0.0:4000/vertex_ai/discovery/v1/projects/fake-project/locations/global/dataStores/test-store-123/servingConfigs/default_config:search \
-  -H 'Authorization: Bearer YOUR_LITELLM_API_KEY' \
-  -H 'Content-Type: application/json' \
-  -d '{"query": "test query"}'
-```
-
-The proxy will use your environment credentials to make the request to Vertex AI!
-

batch_small.jsonl

@@ -1,4 +0,0 @@
-{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello, how are you?"}]}}
-{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "What is the weather today?"}]}}
-{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Tell me a short joke"}]}}
-

ci_cd/.grype.yaml

@@ -0,0 +1,36 @@
+ignore:
+  - vulnerability: CVE-2026-22184
+    reason: no fixed zlib package is available yet in the Wolfi repositories, so this is ignored temporarily until an upstream release exists
+  # Wolfi base image: Python 3.13 and Node from apk have no fixed builds in Wolfi yet / not applicable
+  - vulnerability: CVE-2025-55130
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-59465
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-55131
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-59466
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2026-21637
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: CVE-2025-55132
+    reason: Node in Wolfi apk; only used for Admin UI build/prisma
+  - vulnerability: GHSA-hx9q-6w63-j58v
+    reason: orjson dumps recursion; allowlisted
+  - vulnerability: GHSA-73rr-hh4g-fpgx
+    reason: diff npm transitive dep; override in package.json, allowlisted
+  - vulnerability: CVE-2026-0865
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-15282
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2026-0672
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-15366
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-15367
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-11468
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2025-12781
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet
+  - vulnerability: CVE-2026-1299
+    reason: Python 3.13 in Wolfi base; no fixed apk build yet

ci_cd/TEST_KEY_PATTERNS.md

@@ -0,0 +1,40 @@
+# Test Key Patterns Standard
+
+Standard patterns for test/mock keys and credentials in the LiteLLM codebase to avoid triggering secret detection.
+
+## How GitGuardian Works
+
+GitGuardian uses **machine learning and entropy analysis**, not just pattern matching:
+- **Low entropy** values (like `sk-1234`, `postgres`) are automatically ignored
+- **High entropy** values (realistic-looking secrets) trigger detection
+- **Context-aware** detection understands code syntax like `os.environ["KEY"]`
+
+## Recommended Test Key Patterns
+
+### Option 1: Low Entropy Values (Simplest)
+These won't trigger GitGuardian's ML detector:
+
+```python
+api_key = "sk-1234"
+api_key = "sk-12345"
+database_password = "postgres"
+token = "test123"
+```
+
+### Option 2: High Entropy with Test Prefixes
+If you need realistic-looking test keys with high entropy, use these prefixes:
+
+```python
+api_key = "sk-test-abc123def456ghi789..."  # OpenAI-style test key
+api_key = "sk-mock-1234567890abcdef1234..."  # Mock key
+api_key = "sk-fake-xyz789uvw456rst123..."  # Fake key
+token = "test-api-key-with-high-entropy"
+```
+
+## Configured Ignore Patterns
+
+These patterns are in `.gitguardian.yaml` for high-entropy test keys:
+- `sk-test-*` - OpenAI-style test keys
+- `sk-mock-*` - Mock API keys  
+- `sk-fake-*` - Fake API keys
+- `test-api-key` - Generic test tokens

ci_cd/security_scans.sh

@@ -26,15 +26,65 @@ install_grype() {
     echo "Grype installed successfully"
 }
 
+# Function to install ggshield
+install_ggshield() {
+    echo "Installing ggshield..."
+    pip3 install --upgrade pip
+    pip3 install ggshield
+    echo "ggshield installed successfully"
+}
+
+# # Function to run secret detection scans
+# run_secret_detection() {
+#     echo "Running secret detection scans..."
+    
+#     if ! command -v ggshield &> /dev/null; then
+#         install_ggshield
+#     fi
+    
+#     # Check if GITGUARDIAN_API_KEY is set (required for CI/CD)
+#     if [ -z "$GITGUARDIAN_API_KEY" ]; then
+#         echo "Warning: GITGUARDIAN_API_KEY environment variable is not set."
+#         echo "ggshield requires a GitGuardian API key to scan for secrets."
+#         echo "Please set GITGUARDIAN_API_KEY in your CI/CD environment variables."
+#         exit 1
+#     fi
+    
+#     echo "Scanning codebase for secrets..."
+#     echo "Note: Large codebases may take several minutes due to API rate limits (50 requests/minute on free plan)"
+#     echo "ggshield will automatically handle rate limits and retry as needed."
+#     echo "Binary files, cache files, and build artifacts are excluded via .gitguardian.yaml"
+    
+#     # Use --recursive for directory scanning and auto-confirm if prompted
+#     # .gitguardian.yaml will automatically exclude binary files, wheel files, etc.
+#     # GITGUARDIAN_API_KEY environment variable will be used for authentication
+#     echo y | ggshield secret scan path . --recursive || {
+#         echo ""
+#         echo "=========================================="
+#         echo "ERROR: Secret Detection Failed"
+#         echo "=========================================="
+#         echo "ggshield has detected secrets in the codebase."
+#         echo "Please review discovered secrets above, revoke any actively used secrets"
+#         echo "from underlying systems and make changes to inject secrets dynamically at runtime."
+#         echo ""
+#         echo "For more information, see: https://docs.gitguardian.com/secrets-detection/"
+#         echo "=========================================="
+#         echo ""
+#         exit 1
+#     }
+    
+#     echo "Secret detection scans completed successfully"
+# }
+
 # Function to run Trivy scans
 run_trivy_scans() {
     echo "Running Trivy scans..."
     
     echo "Scanning LiteLLM Docs..."
-    trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
+    trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./docs/
     
     echo "Scanning LiteLLM UI..."
-    trivy fs --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
+    trivy fs --ignorefile .trivyignore --scanners vuln --dependency-tree --exit-code 1 --severity HIGH,CRITICAL,MEDIUM ./ui/
     
     echo "Trivy scans completed successfully"
 }
@@ -51,12 +101,12 @@ run_grype_scans() {
     # Build and scan Dockerfile.database
     echo "Building and scanning Dockerfile.database..."
     docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
-    grype litellm-database:latest --fail-on critical
+    grype litellm-database:latest --config ci_cd/.grype.yaml --fail-on critical
     
     # Build and scan main Dockerfile
     echo "Building and scanning main Dockerfile..."
     docker build --no-cache -t litellm:latest .
-    grype litellm:latest --fail-on critical
+    grype litellm:latest --config ci_cd/.grype.yaml --fail-on critical
     
     # Restore original .dockerignore
     echo "Restoring original .dockerignore..."
@@ -69,10 +119,50 @@ run_grype_scans() {
     # Allowlist of CVEs to be ignored in failure threshold/reporting
     # - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
     # - GHSA-4xh5-x5gv-qwph: GitHub Security Advisory alias for CVE-2025-8869
+    # - GHSA-5j98-mcp5-4vw2: glob CLI command injection via -c/--cmd; glob CLI is not used in the litellm runtime image,
+    #   and the vulnerable versions are pulled in only via OS-level/node tooling outside of our application code
     ALLOWED_CVES=(
         "CVE-2025-8869"
         "GHSA-4xh5-x5gv-qwph"
         "CVE-2025-8291" # no fix available as of Oct 11, 2025
+        "GHSA-5j98-mcp5-4vw2"
+        "CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
+        "CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
+        "CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
+        "CVE-2026-0861" # Wolfi glibc still flagged even on 2.42-r5; upstream patched build unavailable yet
+        "CVE-2010-4756" # glibc glob DoS - awaiting patched Wolfi glibc build
+        "CVE-2019-1010022" # glibc stack guard bypass - awaiting patched Wolfi glibc build
+        "CVE-2019-1010023" # glibc ldd remap issue - awaiting patched Wolfi glibc build
+        "CVE-2019-1010024" # glibc ASLR mitigation bypass - awaiting patched Wolfi glibc build
+        "CVE-2019-1010025" # glibc pthread heap address leak - awaiting patched Wolfi glibc build
+        "CVE-2026-22184" # zlib untgz buffer overflow - untgz unused + no fixed Wolfi build yet
+        "GHSA-58pv-8j8x-9vj2" # jaraco.context path traversal - setuptools vendored only (v5.3.0), not used in application code (using v6.1.0+)
+        "GHSA-34x7-hfp2-rc4v" # node-tar hardlink path traversal - not applicable, tar CLI not exposed in application code
+        "GHSA-r6q2-hw4h-h46w" # node-tar not used by application runtime, Linux-only container, not affect by macOS APFS-specific exploit
+        "GHSA-8rrh-rw8j-w5fx" # wheel is from chainguard and will be handled by then TODO: Remove this after Chainguard updates the wheel
+        "CVE-2025-59465" # Node only used for Admin UI build/prisma
+        "CVE-2025-55131" # Node only used for Admin UI build/prisma
+        "CVE-2025-59466" # Node only used for Admin UI build/prisma
+        "CVE-2025-55130" # Node only used for Admin UI build/prisma
+        "CVE-2025-59467" # Node only used for Admin UI build/prisma
+        "CVE-2026-21637" # Node only used for Admin UI build/prisma
+        "CVE-2025-55132" # Node only used for Admin UI build/prisma
+        "GHSA-hx9q-6w63-j58v" # orjson dumps recursion; allowlisted
+        "CVE-2025-15281" # No fix available yet
+        "CVE-2026-0865" # No fix available yet
+        "CVE-2025-15282" # No fix available yet
+        "CVE-2026-0672" # No fix available yet
+        "CVE-2025-15366" # No fix available yet
+        "CVE-2025-15367" # No fix available yet
+        "CVE-2025-12781" # No fix available yet
+        "CVE-2025-11468" # No fix available yet
+        "CVE-2026-1299" # Python 3.13 email module header injection - not applicable, LiteLLM doesn't use BytesGenerator for email serialization
+        "CVE-2026-0775" # npm cli incorrect permission assignment - no fix available yet, npm is only used at build/prisma-generate time
+        "GHSA-3ppc-4f35-3m26" # minimatch ReDoS via repeated wildcards - from nodejs_wheel bundled npm, not used in application runtime code
+        "GHSA-83g3-92jg-28cx" # tar arbitrary file read/write via hardlink - from nodejs_wheel bundled npm, not used in application runtime code
+        "CVE-2026-25639" # axios - full fix requires 1.x major version bump; pinned to >=0.30.2 to clear other axios CVEs, upgrade to 1.x in follow-up
+        "CVE-2026-2297" # Python 3.13 SourcelessFileLoader audit hook bypass - no fix available in base image
+        "GHSA-qffp-2rhf-9h96" # tar hardlink path traversal - from nodejs_wheel bundled npm, not used in application runtime code
     )
 
     # Build JSON array of allowlisted CVE IDs for jq
@@ -153,6 +243,9 @@ main() {
     install_trivy
     install_grype
     
+    # echo "Running secret detection scans..."
+    # run_secret_detection
+    
     echo "Running filesystem vulnerability scans..."
     run_trivy_scans
     

cookbook/LiteLLM_CometAPI.ipynb

@@ -28,7 +28,7 @@
       "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.6.1)\n",
       "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.1.6)\n",
       "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (4.25.1)\n",
-      "Requirement already satisfied: openai>=1.99.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n",
+      "Requirement already satisfied: openai>=2.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n",
       "Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (2.11.10)\n",
       "Requirement already satisfied: python-dotenv>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.1.1)\n",
       "Requirement already satisfied: tiktoken>=0.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.12.0)\n",
@@ -50,11 +50,11 @@
       "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2025.9.1)\n",
       "Requirement already satisfied: referencing>=0.28.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.36.2)\n",
       "Requirement already satisfied: rpds-py>=0.7.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.27.1)\n",
-      "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.9.0)\n",
-      "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (0.11.0)\n",
-      "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (1.3.1)\n",
-      "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.67.1)\n",
-      "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=1.99.5->litellm) (4.15.0)\n",
+      "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.9.0)\n",
+      "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (0.11.0)\n",
+      "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.3.1)\n",
+      "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.67.1)\n",
+      "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.15.0)\n",
       "Requirement already satisfied: annotated-types>=0.6.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.7.0)\n",
       "Requirement already satisfied: pydantic-core==2.33.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (2.33.2)\n",
       "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.4.2)\n",

cookbook/LiteLLM_HuggingFace.ipynb

@@ -131,7 +131,7 @@
         "            {\n",
         "                \"type\": \"image_url\",\n",
         "                \"image_url\": {\n",
-        "                    \"url\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg\",\n",
+        "                    \"url\": \"https://awsmp-logos.s3.amazonaws.com/seller-xw5kijmvmzasy/c233c9ade2ccb5491072ae232c814942.png\",\n",
         "                },\n",
         "            },\n",
         "        ],\n",

cookbook/LiteLLM_PromptLayer.ipynb

@@ -39,7 +39,7 @@
         "import os\n",
         "os.environ['OPENAI_API_KEY'] = \"\"\n",
         "os.environ['REPLICATE_API_TOKEN'] = \"\"\n",
-        "os.environ['PROMPTLAYER_API_KEY'] = \"pl_4ea2bb00a4dca1b8a70cebf2e9e11564\"\n",
+        "os.environ['PROMPTLAYER_API_KEY'] = \"test-promptlayer-key-123\"\n",
         "\n",
         "# Set Promptlayer as a success callback\n",
         "litellm.success_callback =['promptlayer']\n",

cookbook/Migrating_to_LiteLLM_Proxy_from_OpenAI_Azure_OpenAI.ipynb

@@ -1,21 +1,10 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": []
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    }
-  },
   "cells": [
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "kccfk0mHZ4Ad"
+      },
       "source": [
         "# Migrating to LiteLLM Proxy from OpenAI/Azure OpenAI\n",
         "\n",
@@ -32,29 +21,26 @@
         "To pass provider-specific args, [go here](https://docs.litellm.ai/docs/completion/provider_specific_params#proxy-usage)\n",
         "\n",
         "To drop unsupported params (E.g. frequency_penalty for bedrock with librechat), [go here](https://docs.litellm.ai/docs/completion/drop_params#openai-proxy-usage)\n"
-      ],
-      "metadata": {
-        "id": "kccfk0mHZ4Ad"
-      }
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "nmSClzCPaGH6"
+      },
       "source": [
         "## /chat/completion\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "nmSClzCPaGH6"
-      }
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### OpenAI Python SDK"
-      ],
       "metadata": {
         "id": "_vqcjwOVaKpO"
-      }
+      },
+      "source": [
+        "### OpenAI Python SDK"
+      ]
     },
     {
       "cell_type": "code",
@@ -94,15 +80,20 @@
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "## Function Calling"
-      ],
       "metadata": {
         "id": "AqkyKk9Scxgj"
-      }
+      },
+      "source": [
+        "## Function Calling"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "wDg10VqLczE1"
+      },
+      "outputs": [],
       "source": [
         "from openai import OpenAI\n",
         "client = OpenAI(\n",
@@ -139,24 +130,24 @@
         ")\n",
         "\n",
         "print(completion)\n"
-      ],
-      "metadata": {
-        "id": "wDg10VqLczE1"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Azure OpenAI Python SDK"
-      ],
       "metadata": {
         "id": "YYoxLloSaNWW"
-      }
+      },
+      "source": [
+        "### Azure OpenAI Python SDK"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "yA1XcgowaSRy"
+      },
+      "outputs": [],
       "source": [
         "import openai\n",
         "client = openai.AzureOpenAI(\n",
@@ -184,24 +175,24 @@
         ")\n",
         "\n",
         "print(response)"
-      ],
-      "metadata": {
-        "id": "yA1XcgowaSRy"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Langchain Python"
-      ],
       "metadata": {
         "id": "yl9qhDvnaTpL"
-      }
+      },
+      "source": [
+        "### Langchain Python"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "5MUZgSquaW5t"
+      },
+      "outputs": [],
       "source": [
         "from langchain.chat_models import ChatOpenAI\n",
         "from langchain.prompts.chat import (\n",
@@ -239,24 +230,22 @@
         "response = chat(messages)\n",
         "\n",
         "print(response)"
-      ],
-      "metadata": {
-        "id": "5MUZgSquaW5t"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Curl"
-      ],
       "metadata": {
         "id": "B9eMgnULbRaz"
-      }
+      },
+      "source": [
+        "### Curl"
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "VWCCk5PFcmhS"
+      },
       "source": [
         "\n",
         "\n",
@@ -280,22 +269,24 @@
         "}'\n",
         "```\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "VWCCk5PFcmhS"
-      }
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### LlamaIndex"
-      ],
       "metadata": {
         "id": "drBAm2e1b6xe"
-      }
+      },
+      "source": [
+        "### LlamaIndex"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "d0bZcv8fb9mL"
+      },
+      "outputs": [],
       "source": [
         "import os, dotenv\n",
         "\n",
@@ -326,24 +317,24 @@
         "query_engine = index.as_query_engine()\n",
         "response = query_engine.query(\"What did the author do growing up?\")\n",
         "print(response)\n"
-      ],
-      "metadata": {
-        "id": "d0bZcv8fb9mL"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Langchain JS"
-      ],
       "metadata": {
         "id": "xypvNdHnb-Yy"
-      }
+      },
+      "source": [
+        "### Langchain JS"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "R55mK2vCcBN2"
+      },
+      "outputs": [],
       "source": [
         "import { ChatOpenAI } from \"@langchain/openai\";\n",
         "\n",
@@ -359,24 +350,24 @@
         "const message = await model.invoke(\"Hi there!\");\n",
         "\n",
         "console.log(message);\n"
-      ],
-      "metadata": {
-        "id": "R55mK2vCcBN2"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### OpenAI JS"
-      ],
       "metadata": {
         "id": "nC4bLifCcCiW"
-      }
+      },
+      "source": [
+        "### OpenAI JS"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "MICH8kIMcFpg"
+      },
+      "outputs": [],
       "source": [
         "const { OpenAI } = require('openai');\n",
         "\n",
@@ -398,24 +389,24 @@
         "}\n",
         "\n",
         "main();\n"
-      ],
-      "metadata": {
-        "id": "MICH8kIMcFpg"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Anthropic SDK"
-      ],
       "metadata": {
         "id": "D1Q07pEAcGTb"
-      }
+      },
+      "source": [
+        "### Anthropic SDK"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "qBjFcAvgcI3t"
+      },
+      "outputs": [],
       "source": [
         "import os\n",
         "\n",
@@ -423,7 +414,7 @@
         "\n",
         "client = Anthropic(\n",
         "    base_url=\"http://localhost:4000\", # proxy endpoint\n",
-        "    api_key=\"sk-s4xN1IiLTCytwtZFJaYQrA\", # litellm proxy virtual key\n",
+        "    api_key=\"sk-test-proxy-key-123\", # litellm proxy virtual key (example)\n",
         ")\n",
         "\n",
         "message = client.messages.create(\n",
@@ -437,33 +428,33 @@
         "    model=\"claude-3-opus-20240229\",\n",
         ")\n",
         "print(message.content)"
-      ],
-      "metadata": {
-        "id": "qBjFcAvgcI3t"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "## /embeddings"
-      ],
       "metadata": {
         "id": "dFAR4AJGcONI"
-      }
+      },
+      "source": [
+        "## /embeddings"
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### OpenAI Python SDK"
-      ],
       "metadata": {
         "id": "lgNoM281cRzR"
-      }
+      },
+      "source": [
+        "### OpenAI Python SDK"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "NY3DJhPfcQhA"
+      },
+      "outputs": [],
       "source": [
         "import openai\n",
         "from openai import OpenAI\n",
@@ -478,24 +469,24 @@
         ")\n",
         "\n",
         "print(response)\n"
-      ],
-      "metadata": {
-        "id": "NY3DJhPfcQhA"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Langchain Embeddings"
-      ],
       "metadata": {
         "id": "hmbg-DW6cUZs"
-      }
+      },
+      "source": [
+        "### Langchain Embeddings"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lX2S8Nl1cWVP"
+      },
+      "outputs": [],
       "source": [
         "from langchain.embeddings import OpenAIEmbeddings\n",
         "\n",
@@ -526,24 +517,22 @@
         "\n",
         "print(f\"TITAN EMBEDDINGS\")\n",
         "print(query_result[:5])"
-      ],
-      "metadata": {
-        "id": "lX2S8Nl1cWVP"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "markdown",
-      "source": [
-        "### Curl Request"
-      ],
       "metadata": {
         "id": "oqGbWBCQcYfd"
-      }
+      },
+      "source": [
+        "### Curl Request"
+      ]
     },
     {
       "cell_type": "markdown",
+      "metadata": {
+        "id": "7rkIMV9LcdwQ"
+      },
       "source": [
         "\n",
         "\n",
@@ -556,10 +545,21 @@
         "  }'\n",
         "```\n",
         "\n"
-      ],
-      "metadata": {
-        "id": "7rkIMV9LcdwQ"
-      }
+      ]
     }
-  ]
-}
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

cookbook/ai_coding_tool_guides/claude_code_quickstart/guide.md

@@ -0,0 +1,295 @@
+# Claude Code with LiteLLM Quickstart
+
+This guide shows how to call Claude models (and any LiteLLM-supported model) through LiteLLM proxy from Claude Code.
+
+> **Note:** This integration is based on [Anthropic's official LiteLLM configuration documentation](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration). It allows you to use any LiteLLM supported model through Claude Code with centralized authentication, usage tracking, and cost controls.
+
+## Video Walkthrough
+
+Watch the full tutorial: https://www.loom.com/embed/3c17d683cdb74d36a3698763cc558f56
+
+## Prerequisites
+
+- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
+- API keys for your chosen providers
+
+## Installation
+
+First, install LiteLLM with proxy support:
+
+```bash
+pip install 'litellm[proxy]'
+```
+
+## Step 1: Setup config.yaml
+
+Create a secure configuration using environment variables:
+
+```yaml
+model_list:
+  # Claude models
+  - model_name: claude-3-5-sonnet-20241022    
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+  
+  - model_name: claude-3-5-haiku-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-haiku-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+Set your environment variables:
+
+```bash
+export ANTHROPIC_API_KEY="your-anthropic-api-key"
+export LITELLM_MASTER_KEY="sk-1234567890"  # Generate a secure key
+```
+
+## Step 2: Start Proxy
+
+```bash
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+## Step 3: Verify Setup
+
+Test that your proxy is working correctly:
+
+```bash
+curl -X POST http://0.0.0.0:4000/v1/messages \
+-H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+-H "Content-Type: application/json" \
+-d '{
+    "model": "claude-3-5-sonnet-20241022",
+    "max_tokens": 1000,
+    "messages": [{"role": "user", "content": "What is the capital of France?"}]
+}'
+```
+
+## Step 4: Configure Claude Code
+
+### Method 1: Unified Endpoint (Recommended)
+
+Configure Claude Code to use LiteLLM's unified endpoint. Either a virtual key or master key can be used here:
+
+```bash
+export ANTHROPIC_BASE_URL="http://0.0.0.0:4000"
+export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
+```
+
+> **Tip:** LITELLM_MASTER_KEY gives Claude access to all proxy models, whereas a virtual key would be limited to the models set in the UI.
+
+### Method 2: Provider-specific Pass-through Endpoint
+
+Alternatively, use the Anthropic pass-through endpoint:
+
+```bash
+export ANTHROPIC_BASE_URL="http://0.0.0.0:4000/anthropic"
+export ANTHROPIC_AUTH_TOKEN="$LITELLM_MASTER_KEY"
+```
+
+## Step 5: Use Claude Code
+
+### Choosing Your Model
+
+You have two options for specifying which model Claude Code uses:
+
+#### Option 1: Command Line / Session Model Selection
+
+Specify the model directly when starting Claude Code or during a session:
+
+```bash
+# Specify model at startup
+claude --model claude-3-5-sonnet-20241022
+
+# Or change model during a session
+/model claude-3-5-haiku-20241022
+```
+
+This method uses the exact model you specify.
+
+#### Option 2: Environment Variables
+
+Configure default models using environment variables:
+
+```bash
+# Tell Claude Code which models to use by default
+export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
+export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
+export ANTHROPIC_DEFAULT_OPUS_MODEL=claude-opus-3-5-20240229
+
+claude  # Will use the models specified above
+```
+
+**Note:** Claude Code may cache the model from a previous session. If environment variables don't take effect, use Option 1 to explicitly set the model.
+
+**Important:** The `model_name` in your LiteLLM config must match what Claude Code requests (either from env vars or command line).
+
+### Using 1M Context Window
+
+Claude Code supports extended context (1 million tokens) using the `[1m]` suffix with Claude 4+ models:
+
+```bash
+# Use Sonnet 4.5 with 1M context (requires quotes for shell)
+claude --model 'claude-sonnet-4-5-20250929[1m]'
+
+# Inside a Claude Code session (no quotes needed)
+/model claude-sonnet-4-5-20250929[1m]
+```
+
+**Important:** When using `--model` with `[1m]` in the shell, you must use quotes to prevent the shell from interpreting the brackets.
+
+Alternatively, set as default with environment variables:
+
+```bash
+export ANTHROPIC_DEFAULT_SONNET_MODEL='claude-sonnet-4-5-20250929[1m]'
+claude
+```
+
+**How it works:**
+- Claude Code strips the `[1m]` suffix before sending to LiteLLM
+- Claude Code automatically adds the header `anthropic-beta: context-1m-2025-08-07`
+- Your LiteLLM config should **NOT** include `[1m]` in model names
+
+**Verify 1M context is active:**
+```bash
+/context
+# Should show: 21k/1000k tokens (2%)
+```
+
+**Pricing:** Models using 1M context have different pricing. Input tokens above 200k are charged at a higher rate.
+
+## Troubleshooting
+
+Common issues and solutions:
+
+**Claude Code not connecting:**
+- Verify your proxy is running: `curl http://0.0.0.0:4000/health`
+- Check that `ANTHROPIC_BASE_URL` is set correctly
+- Ensure your `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
+
+**Authentication errors:**
+- Verify your environment variables are set: `echo $LITELLM_MASTER_KEY`
+- Check that your API keys are valid and have sufficient credits
+- Ensure the `ANTHROPIC_AUTH_TOKEN` matches your LiteLLM master key
+
+**Model not found:**
+- Check what model Claude Code is requesting in LiteLLM logs
+- Ensure your `config.yaml` has a matching `model_name` entry
+- If using environment variables, verify they're set: `echo $ANTHROPIC_DEFAULT_SONNET_MODEL`
+
+**1M context not working (showing 200k instead of 1000k):**
+- Verify you're using the `[1m]` suffix: `/model your-model-name[1m]`
+- Check LiteLLM logs for the header `context-1m-2025-08-07` in the request
+- Ensure your model supports 1M context (only certain Claude models do)
+- Your LiteLLM config should **NOT** include `[1m]` in the `model_name`
+
+## Using Multiple Models and Providers
+
+You can configure LiteLLM to route to any supported provider. Here's an example with multiple providers:
+
+```yaml
+model_list:
+  # OpenAI models
+  - model_name: codex-mini
+    litellm_params:
+      model: openai/codex-mini
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: https://api.openai.com/v1
+
+  - model_name: o3-pro
+    litellm_params:
+      model: openai/o3-pro
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: https://api.openai.com/v1
+
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+      api_base: https://api.openai.com/v1
+
+  # Anthropic models
+  - model_name: claude-3-5-sonnet-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: claude-3-5-haiku-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-haiku-20241022
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  # AWS Bedrock
+  - model_name: claude-bedrock
+    litellm_params:
+      model: bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0
+      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
+      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
+      aws_region_name: us-east-1
+
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+**Note:** The `model_name` can be anything you choose. Claude Code will request whatever model you specify (via env vars or command line), and LiteLLM will route to the `model` configured in `litellm_params`.
+
+Switch between models seamlessly:
+
+```bash
+# Use environment variables to set defaults
+export ANTHROPIC_DEFAULT_SONNET_MODEL=claude-3-5-sonnet-20241022
+export ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-3-5-haiku-20241022
+
+# Or specify directly
+claude --model claude-3-5-sonnet-20241022  # Complex reasoning
+claude --model claude-3-5-haiku-20241022    # Fast responses
+claude --model claude-bedrock                # Bedrock deployment
+```
+
+## Default Models Used by Claude Code
+
+If you **don't** set environment variables, Claude Code uses these default model names:
+
+| Purpose | Default Model Name (v2.1.14) |
+|---------|------------------------------|
+| Main model | `claude-sonnet-4-5-20250929` |
+| Light tasks (subagents, summaries) | `claude-haiku-4-5-20251001` |
+| Planning mode | `claude-opus-4-5-20251101` |
+
+Your LiteLLM config should include these model names if you want Claude Code to work without setting environment variables:
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-5-20250929
+    litellm_params:
+      # Can be any provider - Anthropic, Bedrock, Vertex AI, etc.
+      model: anthropic/claude-sonnet-4-5-20250929
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: claude-haiku-4-5-20251001
+    litellm_params:
+      model: anthropic/claude-haiku-4-5-20251001
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: claude-opus-4-5-20251101
+    litellm_params:
+      model: anthropic/claude-opus-4-5-20251101
+      api_key: os.environ/ANTHROPIC_API_KEY
+```
+
+**Warning:** These default model names may change with new Claude Code versions. Check LiteLLM proxy logs for "model not found" errors to identify what Claude Code is requesting.
+
+## Additional Resources
+
+- [LiteLLM Documentation](https://docs.litellm.ai/)
+- [Claude Code Documentation](https://docs.anthropic.com/en/docs/claude-code/overview)
+- [Anthropic's LiteLLM Configuration Guide](https://docs.anthropic.com/en/docs/claude-code/llm-gateway#litellm-configuration)
+

cookbook/ai_coding_tool_guides/index.json

@@ -0,0 +1,134 @@
+[{
+    "title": "Claude Code Quickstart",
+    "description": "This is a quickstart guide to using Claude Code with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_responses_api",
+    "date": "2026-01-15",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM"
+    ]
+},
+{
+    "title": "Claude Code with MCPs",
+    "description": "This is a guide to using Claude Code with MCPs via LiteLLM Proxy.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_mcp",
+    "date": "2026-01-15",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM",
+        "MCP"
+    ]
+}, 
+{
+    "title": "Claude Code with Non-Anthropic Models",
+    "description": "This is a guide to using Claude Code with non-Anthropic models via LiteLLM Proxy.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_non_anthropic_models",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM",
+        "OpenAI",
+        "Gemini"
+    ]
+},
+{
+    "title": "Cursor Quickstart",
+    "description": "This is a quickstart guide to using Cursor with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/cursor_integration",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Cursor",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "Github Copilot Quickstart",
+    "description": "This is a quickstart guide to using Github Copilot with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/github_copilot_integration",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Github Copilot",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "LiteLLM Gemini CLI Quickstart",
+    "description": "This is a quickstart guide to using LiteLLM Gemini CLI.",
+    "url": "https://docs.litellm.ai/docs/tutorials/litellm_gemini_cli",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "Gemini CLI",
+        "Gemini",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "OpenAI Codex CLI Quickstart",
+    "description": "This is a quickstart guide to using OpenAI Codex CLI.",
+    "url": "https://docs.litellm.ai/docs/tutorials/openai_codex",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "OpenAI Codex CLI",
+        "OpenAI",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "OpenWebUI Quickstart",
+    "description": "This is a quickstart guide to using OpenWebUI with LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/openweb_ui",
+    "date": "2026-01-16",
+    "version": "1.0.0",
+    "tags": [
+        "OpenWebUI",
+        "LiteLLM",
+        "Quickstart"
+    ]
+},
+{
+    "title": "AI Coding Tool Usage Tracking",
+    "description": "This is a guide to tracking usage for AI coding tools monitor the use of Claude Code , Google Antigravity, OpenAI Codex, Roo Code etc. through LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/cost_tracking_coding",
+    "date": "2026-01-17",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "Gemini CLI",
+        "OpenAI Codex",
+        "LiteLLM"
+    ]
+},
+{
+    "title": "Use Web Search with Claude Code (across Bedrock/OpenAI/Gemini/etc.)",
+    "description": "This is a guide for using Web Search with Claude Code via LiteLLM.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_code_websearch",
+    "date": "2026-01-17",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM",
+        "Web Search"
+    ]
+},
+{
+    "title": "Track Claude Code Usage per user via Custom Headers",
+    "description": "This is a guide for tracking claude code user usage by passing a customer ID header.",
+    "url": "https://docs.litellm.ai/docs/tutorials/claude_code_customer_tracking",
+    "date": "2026-01-17",
+    "version": "1.0.0",
+    "tags": [
+        "Claude Code",
+        "LiteLLM"
+    ]
+}]

cookbook/anthropic_agent_sdk/README.md

@@ -0,0 +1,144 @@
+# Claude Agent SDK with LiteLLM Gateway
+
+A simple example showing how to use Claude's Agent SDK with LiteLLM as a proxy. This lets you use any LLM provider (OpenAI, Bedrock, Azure, etc.) through the Agent SDK.
+
+## Quick Start
+
+### 1. Install dependencies
+
+```bash
+pip install anthropic claude-agent-sdk litellm
+```
+
+### 2. Start LiteLLM proxy
+
+```bash
+# Simple start with Claude
+litellm --model claude-sonnet-4-20250514
+
+# Or with a config file
+litellm --config config.yaml
+```
+
+### 3. Run the chat
+
+**Basic Agent (no MCP):**
+
+```bash
+python main.py
+```
+
+**Agent with MCP (DeepWiki2 for research):**
+
+```bash
+python agent_with_mcp.py
+```
+
+If MCP connection fails, you can disable it:
+
+```bash
+USE_MCP=false python agent_with_mcp.py
+```
+
+That's it! You can now chat with the agent in your terminal.
+
+### Chat Commands
+
+While chatting, you can use these commands:
+- `models` - List all available models (fetched from your LiteLLM proxy)
+- `model` - Switch to a different model
+- `clear` - Start a new conversation
+- `quit` or `exit` - End the chat
+
+The chat automatically fetches available models from your LiteLLM proxy's `/models` endpoint, so you'll always see what's currently configured.
+
+## Configuration
+
+Set these environment variables if needed:
+
+```bash
+export LITELLM_PROXY_URL="http://localhost:4000"
+export LITELLM_API_KEY="sk-1234"
+export LITELLM_MODEL="bedrock-claude-sonnet-4.5"
+```
+
+Or just use the defaults - it'll connect to `http://localhost:4000` by default.
+
+## Files
+
+- `main.py` - Basic interactive agent without MCP
+- `agent_with_mcp.py` - Agent with MCP server integration (DeepWiki2)
+- `common.py` - Shared utilities and functions
+- `config.example.yaml` - Example LiteLLM configuration
+- `requirements.txt` - Python dependencies
+
+## Example Config File
+
+If you want to use multiple models, create a `config.yaml` (see `config.example.yaml`):
+
+```yaml
+model_list:
+  - model_name: bedrock-claude-sonnet-4
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+      aws_region_name: "us-east-1"
+```
+
+Then start LiteLLM with: `litellm --config config.yaml`
+
+## How It Works
+
+The key is pointing the Agent SDK to LiteLLM instead of directly to Anthropic:
+
+```python
+# Point to LiteLLM gateway (not Anthropic)
+os.environ["ANTHROPIC_BASE_URL"] = "http://localhost:4000"
+os.environ["ANTHROPIC_API_KEY"] = "sk-1234"  # Your LiteLLM key
+
+# Use any model configured in LiteLLM
+options = ClaudeAgentOptions(
+    model="bedrock-claude-sonnet-4",  # or gpt-4, or anything else
+    system_prompt="You are a helpful assistant.",
+    max_turns=50,
+)
+```
+
+Note: Don't add `/anthropic` to the base URL - LiteLLM handles the routing automatically.
+
+## Why Use This?
+
+- **Switch providers easily**: Use the same code with OpenAI, Bedrock, Azure, etc.
+- **Cost tracking**: LiteLLM tracks spending across all your agent conversations
+- **Rate limiting**: Set budgets and limits on your agent usage
+- **Load balancing**: Distribute requests across multiple API keys or regions
+- **Fallbacks**: Automatically retry with a different model if one fails
+
+## Troubleshooting
+
+**Connection errors?**
+- Make sure LiteLLM is running: `litellm --model your-model`
+- Check the URL is correct (default: `http://localhost:4000`)
+
+**Authentication errors?**
+- Verify your LiteLLM API key is correct
+- Make sure the model is configured in your LiteLLM setup
+
+**Model not found?**
+- Check the model name matches what's in your LiteLLM config
+- Run `litellm --model your-model` to test it works
+
+**Agent with MCP stuck or failing?**
+- The MCP server might not be available at `http://localhost:4000/mcp/deepwiki2`
+- Try disabling MCP: `USE_MCP=false python agent_with_mcp.py`
+- Or use the basic agent: `python main.py`
+
+## Learn More
+
+- [LiteLLM Docs](https://docs.litellm.ai/)
+- [Claude Agent SDK](https://github.com/anthropics/anthropic-agent-sdk)
+- [LiteLLM Proxy Guide](https://docs.litellm.ai/docs/proxy/quick_start)

cookbook/anthropic_agent_sdk/agent_with_mcp.py

@@ -0,0 +1,140 @@
+"""
+Interactive Claude Agent SDK CLI with MCP Support
+
+This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy,
+with MCP (Model Context Protocol) server integration for enhanced capabilities.
+"""
+
+import asyncio
+import os
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+from common import (
+    Config,
+    fetch_available_models,
+    setup_litellm_env,
+    print_header,
+    handle_model_list,
+    handle_model_switch,
+    stream_response,
+)
+
+
+async def interactive_chat_with_mcp():
+    """
+    Interactive CLI chat with the agent and MCP server
+    """
+    config = Config()
+    
+    # Configure Anthropic SDK to point to LiteLLM gateway
+    litellm_base_url = setup_litellm_env(config)
+    
+    # Fetch available models from proxy
+    available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
+    
+    current_model = config.LITELLM_MODEL
+    
+    # MCP server configuration
+    mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
+    use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
+    
+    if not use_mcp:
+        print("⚠️  MCP disabled via USE_MCP=false")
+    
+    print_header(litellm_base_url, current_model, has_mcp=use_mcp)
+    
+    while True:
+        # Configure agent options
+        if use_mcp:
+            try:
+                # Try with MCP server (HTTP transport)
+                # Using McpHttpServerConfig format from Agent SDK
+                options = ClaudeAgentOptions(
+                    system_prompt="You are a helpful AI assistant with access to DeepWiki for research. Be concise, accurate, and friendly.",
+                    model=current_model,
+                    max_turns=50,
+                    mcp_servers={
+                        "deepwiki2": {
+                            "type": "http",
+                            "url": mcp_server_url,
+                            "headers": {
+                                "Authorization": f"Bearer {config.LITELLM_API_KEY}"
+                            }
+                        }
+                    },
+                )
+            except Exception as e:
+                print(f"⚠️  Warning: Could not configure MCP server: {e}")
+                print("Continuing without MCP...\n")
+                use_mcp = False
+                options = ClaudeAgentOptions(
+                    system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+                    model=current_model,
+                    max_turns=50,
+                )
+        else:
+            # Without MCP
+            options = ClaudeAgentOptions(
+                system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+                model=current_model,
+                max_turns=50,
+            )
+        
+        # Create agent client
+        try:
+            async with ClaudeSDKClient(options=options) as client:
+                conversation_active = True
+                
+                while conversation_active:
+                    # Get user input
+                    try:
+                        user_input = input("\n👤 You: ").strip()
+                    except (EOFError, KeyboardInterrupt):
+                        print("\n\n👋 Goodbye!")
+                        return
+                    
+                    # Handle commands
+                    if user_input.lower() in ['quit', 'exit']:
+                        print("\n👋 Goodbye!")
+                        return
+                    
+                    if user_input.lower() == 'clear':
+                        print("\n🔄 Starting new conversation...\n")
+                        conversation_active = False
+                        continue
+                    
+                    if user_input.lower() == 'models':
+                        handle_model_list(available_models, current_model)
+                        continue
+                    
+                    if user_input.lower() == 'model':
+                        new_model, should_restart = handle_model_switch(available_models, current_model)
+                        if should_restart:
+                            current_model = new_model
+                            conversation_active = False
+                        continue
+                    
+                    if not user_input:
+                        continue
+                    
+                    # Stream response from agent
+                    await stream_response(client, user_input)
+        
+        except Exception as e:
+            print(f"\n❌ Error creating agent client: {e}")
+            print("This might be an MCP configuration issue. Try running without MCP:")
+            print("  USE_MCP=false python agent_with_mcp.py")
+            print("\nOr use the basic agent:")
+            print("  python main.py")
+            return
+
+
+def main():
+    """Run interactive chat with MCP"""
+    try:
+        asyncio.run(interactive_chat_with_mcp())
+    except KeyboardInterrupt:
+        print("\n\n👋 Goodbye!")
+
+
+if __name__ == "__main__":
+    main()

cookbook/anthropic_agent_sdk/common.py

@@ -0,0 +1,160 @@
+"""
+Common utilities for Claude Agent SDK examples
+"""
+
+import os
+import httpx
+
+
+class Config:
+    """Configuration for LiteLLM Gateway connection"""
+    
+    # LiteLLM proxy URL (default to local instance)
+    LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
+    
+    # LiteLLM API key (master key or virtual key)
+    LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
+    
+    # Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
+    LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
+
+
+async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
+    """
+    Fetch available models from LiteLLM proxy /models endpoint
+    """
+    try:
+        async with httpx.AsyncClient() as client:
+            response = await client.get(
+                f"{base_url}/models",
+                headers={"Authorization": f"Bearer {api_key}"},
+                timeout=10.0
+            )
+            response.raise_for_status()
+            data = response.json()
+            return [model["id"] for model in data.get("data", [])]
+    except Exception as e:
+        print(f"⚠️  Warning: Could not fetch models from proxy: {e}")
+        print("Using default model list...")
+        # Fallback to default models
+        return [
+            "bedrock-claude-sonnet-3.5",
+            "bedrock-claude-sonnet-4",
+            "bedrock-claude-sonnet-4.5",
+            "bedrock-claude-opus-4.5",
+            "bedrock-nova-premier",
+        ]
+
+
+def setup_litellm_env(config: Config):
+    """
+    Configure environment variables to point Agent SDK to LiteLLM
+    """
+    litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
+    os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
+    os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
+    return litellm_base_url
+
+
+def print_header(base_url: str, current_model: str, has_mcp: bool = False):
+    """
+    Print the chat header
+    """
+    mcp_indicator = " + MCP" if has_mcp else ""
+    print("=" * 70)
+    print(f"🤖 Claude Agent SDK with LiteLLM Gateway{mcp_indicator} - Interactive Chat")
+    print("=" * 70)
+    print(f"🚀 Connected to: {base_url}")
+    print(f"📦 Current model: {current_model}")
+    if has_mcp:
+        print("🔌 MCP: deepwiki2 enabled")
+    print("\nType your messages below. Commands:")
+    print("  - 'quit' or 'exit' to end the conversation")
+    print("  - 'clear' to start a new conversation")
+    print("  - 'model' to switch models")
+    print("  - 'models' to list available models")
+    print("=" * 70)
+    print()
+
+
+def handle_model_list(available_models: list[str], current_model: str):
+    """
+    Display available models
+    """
+    print("\n📋 Available models:")
+    for i, model in enumerate(available_models, 1):
+        marker = "✓" if model == current_model else " "
+        print(f"  {marker} {i}. {model}")
+
+
+def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
+    """
+    Handle model switching
+    
+    Returns:
+        tuple: (new_model, should_restart_conversation)
+    """
+    print("\n📋 Select a model:")
+    for i, model in enumerate(available_models, 1):
+        marker = "✓" if model == current_model else " "
+        print(f"  {marker} {i}. {model}")
+    
+    try:
+        choice = input("\nEnter number (or press Enter to cancel): ").strip()
+        if choice:
+            idx = int(choice) - 1
+            if 0 <= idx < len(available_models):
+                new_model = available_models[idx]
+                print(f"\n✅ Switched to: {new_model}")
+                print("🔄 Starting new conversation with new model...\n")
+                return new_model, True
+            else:
+                print("❌ Invalid choice")
+    except (ValueError, IndexError):
+        print("❌ Invalid input")
+    
+    return current_model, False
+
+
+async def stream_response(client, user_input: str):
+    """
+    Stream response from the agent
+    """
+    print("\n🤖 Assistant: ", end='', flush=True)
+    
+    try:
+        await client.query(user_input)
+        
+        # Show loading indicator
+        print("⏳ thinking...", end='', flush=True)
+        
+        # Stream the response
+        first_chunk = True
+        async for msg in client.receive_response():
+            # Clear loading indicator on first message
+            if first_chunk:
+                print("\r🤖 Assistant: ", end='', flush=True)
+                first_chunk = False
+            
+            # Handle different message types
+            if hasattr(msg, 'type'):
+                if msg.type == 'content_block_delta':
+                    # Streaming text delta
+                    if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
+                        print(msg.delta.text, end='', flush=True)
+                elif msg.type == 'content_block_start':
+                    # Start of content block
+                    if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
+                        print(msg.content_block.text, end='', flush=True)
+            
+            # Fallback to original content handling
+            if hasattr(msg, 'content'):
+                for content_block in msg.content:
+                    if hasattr(content_block, 'text'):
+                        print(content_block.text, end='', flush=True)
+        
+        print()  # New line after response
+        
+    except Exception as e:
+        print(f"\r\n❌ Error: {e}")
+        print("Please check your LiteLLM gateway is running and configured correctly.")

cookbook/anthropic_agent_sdk/config.example.yaml

@@ -0,0 +1,25 @@
+model_list:
+  - model_name: bedrock-claude-sonnet-3.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-20250514-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-sonnet-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-claude-opus-4.5
+    litellm_params:
+      model: "bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0"
+      aws_region_name: "us-east-1"
+
+  - model_name: bedrock-nova-premier
+    litellm_params:
+      model: "bedrock/amazon.nova-premier-v1:0"
+      aws_region_name: "us-east-1"

cookbook/anthropic_agent_sdk/main.py

@@ -0,0 +1,95 @@
+"""
+Simple Interactive Claude Agent SDK CLI using LiteLLM Gateway
+
+This example demonstrates an interactive CLI chat with the Anthropic Agent SDK using LiteLLM as a proxy.
+LiteLLM acts as a unified interface, allowing you to use any LLM provider (OpenAI, Azure, Bedrock, etc.)
+through the Claude Agent SDK by pointing it to the LiteLLM gateway.
+"""
+
+import asyncio
+from claude_agent_sdk import ClaudeSDKClient, ClaudeAgentOptions
+from common import (
+    Config,
+    fetch_available_models,
+    setup_litellm_env,
+    print_header,
+    handle_model_list,
+    handle_model_switch,
+    stream_response,
+)
+
+
+async def interactive_chat():
+    """
+    Interactive CLI chat with the agent
+    """
+    config = Config()
+    
+    # Configure Anthropic SDK to point to LiteLLM gateway
+    litellm_base_url = setup_litellm_env(config)
+    
+    # Fetch available models from proxy
+    available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
+    
+    current_model = config.LITELLM_MODEL
+    
+    print_header(litellm_base_url, current_model)
+    
+    while True:
+        # Configure agent options for each conversation
+        options = ClaudeAgentOptions(
+            system_prompt="You are a helpful AI assistant. Be concise, accurate, and friendly.",
+            model=current_model,
+            max_turns=50,
+        )
+        
+        # Create agent client
+        async with ClaudeSDKClient(options=options) as client:
+            conversation_active = True
+            
+            while conversation_active:
+                # Get user input
+                try:
+                    user_input = input("\n👤 You: ").strip()
+                except (EOFError, KeyboardInterrupt):
+                    print("\n\n👋 Goodbye!")
+                    return
+                
+                # Handle commands
+                if user_input.lower() in ['quit', 'exit']:
+                    print("\n👋 Goodbye!")
+                    return
+                
+                if user_input.lower() == 'clear':
+                    print("\n🔄 Starting new conversation...\n")
+                    conversation_active = False
+                    continue
+                
+                if user_input.lower() == 'models':
+                    handle_model_list(available_models, current_model)
+                    continue
+                
+                if user_input.lower() == 'model':
+                    new_model, should_restart = handle_model_switch(available_models, current_model)
+                    if should_restart:
+                        current_model = new_model
+                        conversation_active = False
+                    continue
+                
+                if not user_input:
+                    continue
+                
+                # Stream response from agent
+                await stream_response(client, user_input)
+
+
+def main():
+    """Run interactive chat"""
+    try:
+        asyncio.run(interactive_chat())
+    except KeyboardInterrupt:
+        print("\n\n👋 Goodbye!")
+
+
+if __name__ == "__main__":
+    main()

cookbook/anthropic_agent_sdk/requirements.txt

@@ -0,0 +1,2 @@
+claude-agent-sdk
+httpx>=0.27.0

cookbook/benchmark/readme.md

@@ -178,4 +178,4 @@ Benchmark Results for 'When will BerriAI IPO?':
 ```
 
 ## Support 
-**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.
+**🤝 Schedule a 1-on-1 Session:** Book a [1-on-1 session](https://calendly.com/d/cx9p-5yf-2nm/litellm-introductions) with Krrish and Ishaan, the founders, to discuss any issues, provide feedback, or explore how we can improve LiteLLM for you.

cookbook/gollem_go_agent_framework/README.md

@@ -0,0 +1,119 @@
+# Gollem Go Agent Framework with LiteLLM
+
+A working example showing how to use [gollem](https://github.com/fugue-labs/gollem), a production-grade Go agent framework, with LiteLLM as a proxy gateway. This lets Go developers access 100+ LLM providers through a single proxy while keeping compile-time type safety for tools and structured output.
+
+## Quick Start
+
+### 1. Start LiteLLM Proxy
+
+```bash
+# Simple start with a single model
+litellm --model gpt-4o
+
+# Or with the example config for multi-provider access
+litellm --config proxy_config.yaml
+```
+
+### 2. Run the examples
+
+```bash
+# Install Go dependencies
+go mod tidy
+
+# Basic agent
+go run ./basic
+
+# Agent with type-safe tools
+go run ./tools
+
+# Streaming responses
+go run ./streaming
+```
+
+## Configuration
+
+The included `proxy_config.yaml` sets up three providers through LiteLLM:
+
+```yaml
+model_list:
+  - model_name: gpt-4o          # OpenAI
+  - model_name: claude-sonnet    # Anthropic
+  - model_name: gemini-pro       # Google Vertex AI
+```
+
+Switch providers in Go by changing a single string — no code changes needed:
+
+```go
+model := openai.NewLiteLLM("http://localhost:4000",
+    openai.WithModel("gpt-4o"),        // OpenAI
+    // openai.WithModel("claude-sonnet"),  // Anthropic
+    // openai.WithModel("gemini-pro"),     // Google
+)
+```
+
+## Examples
+
+### `basic/` — Basic Agent
+
+Connects gollem to LiteLLM and runs a simple prompt. Demonstrates the `NewLiteLLM` constructor and basic agent creation.
+
+### `tools/` — Type-Safe Tools
+
+Shows gollem's compile-time type-safe tool framework working through LiteLLM's tool-use passthrough. The tool parameters are Go structs with JSON tags — the schema is generated automatically at compile time.
+
+### `streaming/` — Streaming Responses
+
+Real-time token streaming using Go 1.23+ range-over-function iterators, proxied through LiteLLM's SSE passthrough.
+
+## How It Works
+
+Gollem's `openai.NewLiteLLM()` constructor creates an OpenAI-compatible provider pointed at your LiteLLM proxy. Since LiteLLM speaks the OpenAI API protocol, everything works out of the box:
+
+- **Chat completions** — standard request/response
+- **Tool use** — LiteLLM passes tool definitions and calls through transparently
+- **Streaming** — Server-Sent Events proxied through LiteLLM
+- **Structured output** — JSON schema response format works with supporting models
+
+```
+Go App (gollem) → LiteLLM Proxy → OpenAI / Anthropic / Google / ...
+```
+
+## Why Use This?
+
+- **Type-safe Go**: Compile-time type checking for tools, structured output, and agent configuration — no runtime surprises
+- **Single proxy, many models**: Switch between OpenAI, Anthropic, Google, and 100+ other providers by changing a model name string
+- **Zero-dependency core**: gollem's core has no external dependencies — just stdlib
+- **Single binary deployment**: `go build` produces one binary, no pip/venv/Docker needed
+- **Cost tracking & rate limiting**: LiteLLM handles cost tracking, rate limits, and fallbacks at the proxy layer
+
+## Environment Variables
+
+```bash
+# Required for providers you want to use (set in LiteLLM config or env)
+export OPENAI_API_KEY="sk-..."
+export ANTHROPIC_API_KEY="sk-ant-..."
+
+# Optional: point to a non-default LiteLLM proxy
+export LITELLM_PROXY_URL="http://localhost:4000"
+```
+
+## Troubleshooting
+
+**Connection errors?**
+- Make sure LiteLLM is running: `litellm --model gpt-4o`
+- Check the URL is correct (default: `http://localhost:4000`)
+
+**Model not found?**
+- Verify the model name matches what's configured in LiteLLM
+- Run `curl http://localhost:4000/models` to see available models
+
+**Tool calls not working?**
+- Ensure the underlying model supports tool use (GPT-4o, Claude, Gemini)
+- Check LiteLLM logs for any provider-specific errors
+
+## Learn More
+
+- [gollem GitHub](https://github.com/fugue-labs/gollem)
+- [gollem API Reference](https://pkg.go.dev/github.com/fugue-labs/gollem/core)
+- [LiteLLM Proxy Docs](https://docs.litellm.ai/docs/simple_proxy)
+- [LiteLLM Supported Models](https://docs.litellm.ai/docs/providers)

cookbook/gollem_go_agent_framework/basic/main.go

@@ -0,0 +1,41 @@
+// Basic gollem agent connected to a LiteLLM proxy.
+//
+// Usage:
+//
+//	litellm --model gpt-4o   # start proxy in another terminal
+//	go run ./basic
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	// Connect to LiteLLM proxy. NewLiteLLM creates an OpenAI-compatible
+	// provider pointed at the given URL.
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"), // any model name configured in LiteLLM
+	)
+
+	// Create and run a simple agent.
+	agent := core.NewAgent[string](model,
+		core.WithSystemPrompt[string]("You are a helpful assistant. Be concise."),
+	)
+
+	result, err := agent.Run(context.Background(), "Explain quantum computing in two sentences.")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(result.Output)
+}

cookbook/gollem_go_agent_framework/go.mod

@@ -0,0 +1,5 @@
+module github.com/BerriAI/litellm/cookbook/gollem_go_agent_framework
+
+go 1.25.1
+
+require github.com/fugue-labs/gollem v0.1.0

cookbook/gollem_go_agent_framework/go.sum

@@ -0,0 +1,2 @@
+github.com/fugue-labs/gollem v0.1.0 h1:QexYnvkb44QZFEljgAePqMIGZjgsbk0Y5GJ2jYYgfa8=
+github.com/fugue-labs/gollem v0.1.0/go.mod h1:htW1YO81uysSKVOkYJtxhGCFrzm+36HBFxEWuECoHKQ=

cookbook/gollem_go_agent_framework/proxy_config.yaml

@@ -0,0 +1,16 @@
+model_list:
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+
+  - model_name: claude-sonnet
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  - model_name: gemini-pro
+    litellm_params:
+      model: vertex_ai/gemini-2.0-flash
+      vertex_project: my-project
+      vertex_location: us-central1

cookbook/gollem_go_agent_framework/streaming/main.go

@@ -0,0 +1,56 @@
+// Streaming responses from gollem through LiteLLM.
+//
+// Uses Go 1.23+ range-over-function iterators for real-time token
+// streaming via LiteLLM's SSE passthrough.
+//
+// Usage:
+//
+//	litellm --model gpt-4o
+//	go run ./streaming
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"),
+	)
+
+	agent := core.NewAgent[string](model)
+
+	// RunStream returns a streaming result that yields tokens as they arrive.
+	stream, err := agent.RunStream(context.Background(), "Write a haiku about distributed systems")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// StreamText yields text chunks in real-time.
+	// The boolean argument controls whether deltas (true) or accumulated
+	// text (false) is returned.
+	fmt.Print("Response: ")
+	for text, err := range stream.StreamText(true) {
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Print(text)
+	}
+	fmt.Println()
+
+	// After streaming completes, the final response is available.
+	resp := stream.Response()
+	fmt.Printf("\nTokens used: input=%d, output=%d\n",
+		resp.Usage.InputTokens, resp.Usage.OutputTokens)
+}

cookbook/gollem_go_agent_framework/tools/main.go

@@ -0,0 +1,64 @@
+// Gollem agent with type-safe tools through LiteLLM.
+//
+// The tool parameters are Go structs — gollem generates the JSON schema
+// automatically at compile time. LiteLLM passes tool definitions through
+// transparently to the underlying provider.
+//
+// Usage:
+//
+//	litellm --model gpt-4o
+//	go run ./tools
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fugue-labs/gollem/core"
+	"github.com/fugue-labs/gollem/provider/openai"
+)
+
+// WeatherParams defines the tool's input schema via struct tags.
+// The JSON schema is generated at compile time — no runtime reflection needed.
+type WeatherParams struct {
+	City string `json:"city" description:"City name to get weather for"`
+	Unit string `json:"unit,omitempty" description:"Temperature unit: celsius or fahrenheit"`
+}
+
+func main() {
+	proxyURL := "http://localhost:4000"
+	if u := os.Getenv("LITELLM_PROXY_URL"); u != "" {
+		proxyURL = u
+	}
+
+	model := openai.NewLiteLLM(proxyURL,
+		openai.WithModel("gpt-4o"),
+	)
+
+	// Define a type-safe tool. The function signature enforces correct types.
+	weatherTool := core.FuncTool[WeatherParams](
+		"get_weather",
+		"Get current weather for a city",
+		func(ctx context.Context, p WeatherParams) (string, error) {
+			unit := p.Unit
+			if unit == "" {
+				unit = "fahrenheit"
+			}
+			// In production, call a real weather API here.
+			return fmt.Sprintf("Weather in %s: 72°F (22°C), sunny", p.City), nil
+		},
+	)
+
+	agent := core.NewAgent[string](model,
+		core.WithTools[string](weatherTool),
+		core.WithSystemPrompt[string]("You are a helpful weather assistant. Use the get_weather tool to answer weather questions."),
+	)
+
+	result, err := agent.Run(context.Background(), "What's the weather like in San Francisco and Tokyo?")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(result.Output)
+}

cookbook/litellm_proxy_server/braintrust_prompt_wrapper_README.md

@@ -0,0 +1,279 @@
+# Braintrust Prompt Wrapper for LiteLLM
+
+This directory contains a wrapper server that enables LiteLLM to use prompts from [Braintrust](https://www.braintrust.dev/) through the generic prompt management API.
+
+## Architecture
+
+```
+┌─────────────┐         ┌──────────────────────┐         ┌─────────────┐
+│   LiteLLM   │ ──────> │  Wrapper Server      │ ──────> │  Braintrust │
+│   Client    │         │  (This Server)       │         │     API     │
+└─────────────┘         └──────────────────────┘         └─────────────┘
+    Uses generic             Transforms                    Stores actual
+    prompt manager        Braintrust format              prompt templates
+                         to LiteLLM format
+```
+
+## Components
+
+### 1. Generic Prompt Manager (`litellm/integrations/generic_prompt_management/`)
+
+A generic client that can work with any API implementing the `/beta/litellm_prompt_management` endpoint.
+
+**Expected API Response Format:**
+```json
+{
+  "prompt_id": "string",
+  "prompt_template": [
+    {"role": "system", "content": "You are a helpful assistant"},
+    {"role": "user", "content": "Hello {name}"}
+  ],
+  "prompt_template_model": "gpt-4",
+  "prompt_template_optional_params": {
+    "temperature": 0.7,
+    "max_tokens": 100
+  }
+}
+```
+
+### 2. Braintrust Wrapper Server (`braintrust_prompt_wrapper_server.py`)
+
+A FastAPI server that:
+- Implements the `/beta/litellm_prompt_management` endpoint
+- Fetches prompts from Braintrust API
+- Transforms Braintrust response format to LiteLLM format
+
+## Setup
+
+### Install Dependencies
+
+```bash
+pip install fastapi uvicorn httpx litellm
+```
+
+### Set Environment Variables
+
+```bash
+export BRAINTRUST_API_KEY="your-braintrust-api-key"
+```
+
+## Usage
+
+### Step 1: Start the Wrapper Server
+
+```bash
+python braintrust_prompt_wrapper_server.py
+```
+
+The server will start on `http://localhost:8080` by default.
+
+You can customize the port and host:
+```bash
+export PORT=8000
+export HOST=0.0.0.0
+python braintrust_prompt_wrapper_server.py
+```
+
+### Step 2: Use with LiteLLM
+
+```python
+import litellm
+from litellm.integrations.generic_prompt_management import GenericPromptManager
+
+# Configure the generic prompt manager to use your wrapper server
+generic_config = {
+    "api_base": "http://localhost:8080",
+    "api_key": "your-braintrust-api-key",  # Will be passed to Braintrust
+    "timeout": 30,
+}
+
+# Create the prompt manager
+prompt_manager = GenericPromptManager(**generic_config)
+
+# Use with completion
+response = litellm.completion(
+    model="generic_prompt/gpt-4",
+    prompt_id="your-braintrust-prompt-id",
+    prompt_variables={"name": "World"},  # Variables to substitute
+    messages=[{"role": "user", "content": "Additional message"}]
+)
+
+print(response)
+```
+
+### Step 3: Direct API Testing
+
+You can also test the wrapper API directly:
+
+```bash
+# Test with curl
+curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \
+     "http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID"
+
+# Health check
+curl http://localhost:8080/health
+
+# Service info
+curl http://localhost:8080/
+```
+
+## API Documentation
+
+Once the server is running, visit:
+- Swagger UI: `http://localhost:8080/docs`
+- ReDoc: `http://localhost:8080/redoc`
+
+## Braintrust Format Transformation
+
+The wrapper automatically transforms Braintrust's response format:
+
+**Braintrust API Response:**
+```json
+{
+  "id": "prompt-123",
+  "prompt_data": {
+    "prompt": {
+      "type": "chat",
+      "messages": [
+        {
+          "role": "system",
+          "content": "You are a helpful assistant"
+        }
+      ]
+    },
+    "options": {
+      "model": "gpt-4",
+      "params": {
+        "temperature": 0.7,
+        "max_tokens": 100
+      }
+    }
+  }
+}
+```
+
+**Transformed to LiteLLM Format:**
+```json
+{
+  "prompt_id": "prompt-123",
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant"
+    }
+  ],
+  "prompt_template_model": "gpt-4",
+  "prompt_template_optional_params": {
+    "temperature": 0.7,
+    "max_tokens": 100
+  }
+}
+```
+
+## Supported Parameters
+
+The wrapper automatically maps these Braintrust parameters to LiteLLM:
+
+- `temperature`
+- `max_tokens` / `max_completion_tokens`
+- `top_p`
+- `frequency_penalty`
+- `presence_penalty`
+- `n`
+- `stop`
+- `response_format`
+- `tool_choice`
+- `function_call`
+- `tools`
+
+## Variable Substitution
+
+The generic prompt manager supports simple variable substitution:
+
+```python
+# In your Braintrust prompt:
+# "Hello {name}, welcome to {place}!"
+
+# In your code:
+prompt_variables = {
+    "name": "Alice",
+    "place": "Wonderland"
+}
+
+# Result:
+# "Hello Alice, welcome to Wonderland!"
+```
+
+Supports both `{variable}` and `{{variable}}` syntax.
+
+## Error Handling
+
+The wrapper provides detailed error messages:
+
+- **401**: Missing or invalid Braintrust API token
+- **404**: Prompt not found in Braintrust
+- **502**: Failed to connect to Braintrust API
+- **500**: Error transforming response
+
+## Production Deployment
+
+For production use:
+
+1. **Use HTTPS**: Deploy behind a reverse proxy with SSL
+2. **Authentication**: Add authentication to the wrapper endpoint if needed
+3. **Rate Limiting**: Implement rate limiting to prevent abuse
+4. **Caching**: Consider caching prompt responses
+5. **Monitoring**: Add logging and monitoring
+
+Example with Docker:
+
+```dockerfile
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN pip install fastapi uvicorn httpx
+
+COPY braintrust_prompt_wrapper_server.py .
+
+ENV PORT=8080
+ENV HOST=0.0.0.0
+
+EXPOSE 8080
+
+CMD ["python", "braintrust_prompt_wrapper_server.py"]
+```
+
+## Extending to Other Providers
+
+This pattern can be used with any prompt management provider:
+
+1. Create a wrapper server that implements `/beta/litellm_prompt_management`
+2. Transform the provider's response to LiteLLM format
+3. Use the generic prompt manager to connect
+
+Example providers:
+- Langsmith
+- PromptLayer
+- Humanloop
+- Custom internal systems
+
+## Troubleshooting
+
+### "No Braintrust API token provided"
+- Set `BRAINTRUST_API_KEY` environment variable
+- Or pass token in `Authorization: Bearer TOKEN` header
+
+### "Failed to connect to Braintrust API"
+- Check your internet connection
+- Verify Braintrust API is accessible
+- Check firewall settings
+
+### "Prompt not found"
+- Verify the prompt ID exists in Braintrust
+- Check that your API token has access to the prompt
+
+## License
+
+This wrapper is part of the LiteLLM project and follows the same license.
+

cookbook/litellm_proxy_server/braintrust_prompt_wrapper_server.py

@@ -0,0 +1,274 @@
+"""
+Mock server that implements the /beta/litellm_prompt_management endpoint
+and acts as a wrapper for calling the Braintrust API.
+
+This server transforms Braintrust's prompt API response into the format
+expected by LiteLLM's generic prompt management client.
+
+Usage:
+    python braintrust_prompt_wrapper_server.py
+
+    # Then test with:
+    curl -H "Authorization: Bearer YOUR_BRAINTRUST_TOKEN" \
+         "http://localhost:8080/beta/litellm_prompt_management?prompt_id=YOUR_PROMPT_ID"
+"""
+
+import json
+import os
+from typing import Any, Dict, List, Optional
+
+import httpx
+from fastapi import FastAPI, HTTPException, Header, Query
+from fastapi.responses import JSONResponse
+import uvicorn
+
+
+app = FastAPI(
+    title="Braintrust Prompt Wrapper",
+    description="Wrapper server for Braintrust prompts to work with LiteLLM",
+    version="1.0.0",
+)
+
+
+def transform_braintrust_message(message: Dict[str, Any]) -> Dict[str, str]:
+    """
+    Transform a Braintrust message to LiteLLM format.
+
+    Braintrust message format:
+    {
+        "role": "system",
+        "content": "...",
+        "name": "..." (optional)
+    }
+
+    LiteLLM format:
+    {
+        "role": "system",
+        "content": "..."
+    }
+    """
+    result = {
+        "role": message.get("role", "user"),
+        "content": message.get("content", ""),
+    }
+
+    # Include name if present
+    if "name" in message:
+        result["name"] = message["name"]
+
+    return result
+
+
+def transform_braintrust_response(
+    braintrust_response: Dict[str, Any],
+) -> Dict[str, Any]:
+    """
+    Transform Braintrust API response to LiteLLM prompt management format.
+
+    Braintrust response format:
+    {
+        "objects": [{
+            "id": "prompt_id",
+            "prompt_data": {
+                "prompt": {
+                    "type": "chat",
+                    "messages": [...],
+                    "tools": "..."
+                },
+                "options": {
+                    "model": "gpt-4",
+                    "params": {
+                        "temperature": 0.7,
+                        "max_tokens": 100,
+                        ...
+                    }
+                }
+            }
+        }]
+    }
+
+    LiteLLM format:
+    {
+        "prompt_id": "prompt_id",
+        "prompt_template": [...],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {...}
+    }
+    """
+    # Extract the first object from the objects array if it exists
+    if "objects" in braintrust_response and len(braintrust_response["objects"]) > 0:
+        prompt_object = braintrust_response["objects"][0]
+    else:
+        prompt_object = braintrust_response
+
+    prompt_data = prompt_object.get("prompt_data", {})
+    prompt_info = prompt_data.get("prompt", {})
+    options = prompt_data.get("options", {})
+
+    # Extract messages
+    messages = prompt_info.get("messages", [])
+    transformed_messages = [transform_braintrust_message(msg) for msg in messages]
+
+    # Extract model
+    model = options.get("model")
+
+    # Extract optional parameters
+    params = options.get("params", {})
+    optional_params: Dict[str, Any] = {}
+
+    # Map common parameters
+    param_mapping = {
+        "temperature": "temperature",
+        "max_tokens": "max_tokens",
+        "max_completion_tokens": "max_tokens",  # Alternative name
+        "top_p": "top_p",
+        "frequency_penalty": "frequency_penalty",
+        "presence_penalty": "presence_penalty",
+        "n": "n",
+        "stop": "stop",
+    }
+
+    for braintrust_param, litellm_param in param_mapping.items():
+        if braintrust_param in params:
+            value = params[braintrust_param]
+            if value is not None:
+                optional_params[litellm_param] = value
+
+    # Handle response_format
+    if "response_format" in params:
+        optional_params["response_format"] = params["response_format"]
+
+    # Handle tool_choice
+    if "tool_choice" in params:
+        optional_params["tool_choice"] = params["tool_choice"]
+
+    # Handle function_call
+    if "function_call" in params:
+        optional_params["function_call"] = params["function_call"]
+
+    # Add tools if present
+    if "tools" in prompt_info and prompt_info["tools"]:
+        optional_params["tools"] = prompt_info["tools"]
+
+    # Handle tool_functions from prompt_data
+    if "tool_functions" in prompt_data and prompt_data["tool_functions"]:
+        optional_params["tool_functions"] = prompt_data["tool_functions"]
+
+    return {
+        "prompt_id": prompt_object.get("id"),
+        "prompt_template": transformed_messages,
+        "prompt_template_model": model,
+        "prompt_template_optional_params": optional_params if optional_params else None,
+    }
+
+
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(
+    prompt_id: str = Query(..., description="The Braintrust prompt ID to fetch"),
+    authorization: Optional[str] = Header(
+        None, description="Bearer token for Braintrust API"
+    ),
+) -> JSONResponse:
+    """
+    Fetch a prompt from Braintrust and transform it to LiteLLM format.
+
+    Args:
+        prompt_id: The Braintrust prompt ID
+        authorization: Bearer token for Braintrust API (from header)
+
+    Returns:
+        JSONResponse with the transformed prompt data
+    """
+    # Extract token from Authorization header or environment
+    braintrust_token = None
+    if authorization and authorization.startswith("Bearer "):
+        braintrust_token = authorization.replace("Bearer ", "")
+    else:
+        braintrust_token = os.getenv("BRAINTRUST_API_KEY")
+
+    if not braintrust_token:
+        raise HTTPException(
+            status_code=401,
+            detail="No Braintrust API token provided. Pass via Authorization header or set BRAINTRUST_API_KEY environment variable.",
+        )
+
+    # Call Braintrust API
+    braintrust_url = f"https://api.braintrust.dev/v1/prompt/{prompt_id}"
+    headers = {
+        "Authorization": f"Bearer {braintrust_token}",
+        "Accept": "application/json",
+    }
+    print(f"headers: {headers}")
+    print(f"braintrust_url: {braintrust_url}")
+    print(f"braintrust_token: {braintrust_token}")
+
+    try:
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.get(braintrust_url, headers=headers)
+            response.raise_for_status()
+            braintrust_data = response.json()
+    except httpx.HTTPStatusError as e:
+        raise HTTPException(
+            status_code=e.response.status_code,
+            detail=f"Braintrust API error: {e.response.text}",
+        )
+    except httpx.RequestError as e:
+        raise HTTPException(
+            status_code=502,
+            detail=f"Failed to connect to Braintrust API: {str(e)}",
+        )
+    except json.JSONDecodeError as e:
+        raise HTTPException(
+            status_code=502,
+            detail=f"Failed to parse Braintrust API response: {str(e)}",
+        )
+
+    print(f"braintrust_data: {braintrust_data}")
+    # Transform the response
+    try:
+        transformed_data = transform_braintrust_response(braintrust_data)
+        print(f"transformed_data: {transformed_data}")
+        return JSONResponse(content=transformed_data)
+    except Exception as e:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Failed to transform Braintrust response: {str(e)}",
+        )
+
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint."""
+    return {"status": "healthy", "service": "braintrust-prompt-wrapper"}
+
+
+@app.get("/")
+async def root():
+    """Root endpoint with service information."""
+    return {
+        "service": "Braintrust Prompt Wrapper for LiteLLM",
+        "version": "1.0.0",
+        "endpoints": {
+            "prompt_management": "/beta/litellm_prompt_management?prompt_id=<id>",
+            "health": "/health",
+        },
+        "documentation": "/docs",
+    }
+
+
+def main():
+    """Run the server."""
+    port = int(os.getenv("PORT", "8080"))
+    host = os.getenv("HOST", "0.0.0.0")
+
+    print(f"🚀 Starting Braintrust Prompt Wrapper Server on {host}:{port}")
+    print(f"📚 API Documentation available at http://{host}:{port}/docs")
+    print(
+        f"🔑 Make sure to set BRAINTRUST_API_KEY environment variable or pass token in Authorization header"
+    )
+
+    uvicorn.run(app, host=host, port=port)
+
+
+if __name__ == "__main__":
+    main()

cookbook/livekit_agent_sdk/README.md

@@ -0,0 +1,114 @@
+# LiveKit Voice Agent with LiteLLM Gateway
+
+Simple example showing how to use LiveKit's xAI realtime plugin with LiteLLM as a proxy. This lets you switch between xAI, OpenAI, and Azure realtime APIs without changing your code.
+
+## Quick Start
+
+### 1. Install dependencies
+
+```bash
+pip install livekit-agents[xai] websockets
+```
+
+### 2. Start LiteLLM proxy
+
+```bash
+# With xAI
+export XAI_API_KEY="your-xai-key"
+litellm --config config.yaml --port 4000
+```
+
+### 3. Run the voice agent
+
+```bash
+python main.py
+```
+
+Type your message and get a voice response from Grok!
+
+## Configuration
+
+Set these environment variables if needed:
+
+```bash
+export LITELLM_PROXY_URL="http://localhost:4000"
+export LITELLM_API_KEY="sk-1234"
+export LITELLM_MODEL="grok-voice-agent"
+```
+
+Or use the defaults - connects to `http://localhost:4000` by default.
+
+## Example Config File
+
+Create a `config.yaml` with your realtime models:
+
+```yaml
+model_list:
+  - model_name: grok-voice-agent
+    litellm_params:
+      model: xai/grok-2-vision-1212
+      api_key: os.environ/XAI_API_KEY
+    model_info:
+      mode: realtime
+
+  - model_name: openai-voice-agent
+    litellm_params:
+      model: gpt-4o-realtime-preview
+      api_key: os.environ/OPENAI_API_KEY
+    model_info:
+      mode: realtime
+
+general_settings:
+  master_key: sk-1234
+```
+
+Then start: `litellm --config config.yaml --port 4000`
+
+## How It Works
+
+LiveKit's xAI plugin connects through LiteLLM proxy by setting `base_url`:
+
+```python
+from livekit.plugins import xai
+
+model = xai.realtime.RealtimeModel(
+    voice="ara",
+    api_key="sk-1234",              # LiteLLM proxy key
+    base_url="http://localhost:4000", # Point to LiteLLM
+)
+```
+
+## Switching Providers
+
+Just change the model in your config - no code changes needed:
+
+**xAI Grok:**
+```yaml
+model: xai/grok-2-vision-1212
+```
+
+**OpenAI:**
+```yaml
+model: gpt-4o-realtime-preview
+```
+
+**Azure OpenAI:**
+```yaml
+model: azure/gpt-4o-realtime-preview
+api_base: https://your-endpoint.openai.azure.com/
+```
+
+## Why Use LiteLLM?
+
+- ✅ **Switch providers** without changing agent code
+- ✅ **Cost tracking** across all voice sessions
+- ✅ **Rate limiting** and budgets
+- ✅ **Load balancing** across multiple API keys
+- ✅ **Fallbacks** to backup models
+
+## Learn More
+
+- [LiveKit xAI Realtime Tutorial](/docs/tutorials/livekit_xai_realtime)
+- [xAI Realtime Docs](/docs/providers/xai_realtime)
+- [LiveKit Agents Documentation](https://docs.livekit.io/agents/)
+- [LiteLLM Realtime API](/docs/realtime)

cookbook/livekit_agent_sdk/config.example.yaml

@@ -0,0 +1,21 @@
+model_list:
+  - model_name: grok-voice-agent
+    litellm_params:
+      model: xai/grok-2-vision-1212
+      api_key: os.environ/XAI_API_KEY
+    model_info:
+      mode: realtime
+
+  - model_name: openai-voice-agent
+    litellm_params:
+      model: gpt-4o-realtime-preview
+      api_key: os.environ/OPENAI_API_KEY
+    model_info:
+      mode: realtime
+
+litellm_settings:
+  drop_params: True
+  telemetry: False
+
+general_settings:
+  master_key: sk-1234  # Change this to a secure key

cookbook/livekit_agent_sdk/main.py

@@ -0,0 +1,112 @@
+"""
+Simple xAI Voice Agent using LiveKit SDK with LiteLLM Gateway
+
+This example shows how to use LiveKit's xAI realtime plugin through LiteLLM proxy.
+LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI, 
+and Azure realtime APIs without changing your agent code.
+"""
+import asyncio
+import json
+import os
+import websockets
+
+# Configuration
+PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
+API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
+MODEL = os.getenv("LITELLM_MODEL", "grok-voice-agent")
+
+
+async def run_voice_agent():
+    """
+    Simple voice agent that:
+    1. Connects to xAI realtime API through LiteLLM proxy
+    2. Sends a user message
+    3. Streams back the response
+    """
+    
+    url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}"
+    headers = {"Authorization": f"Bearer {API_KEY}"}
+    
+    print(f"🎙️  Connecting to voice agent...")
+    print(f"   Model: {MODEL}")
+    print(f"   Proxy: {PROXY_URL}")
+    print()
+    
+    async with websockets.connect(url, additional_headers=headers) as ws:
+        # Receive initial connection event
+        initial = json.loads(await ws.recv())
+        print(f"✅ Connected! Event: {initial['type']}\n")
+        
+        # Get user input
+        user_message = input("💬 Your message: ").strip()
+        if not user_message:
+            user_message = "Tell me a fun fact about AI!"
+        
+        print(f"\n🤖 Sending to {MODEL}...\n")
+        
+        # Send user message
+        await ws.send(json.dumps({
+            "type": "conversation.item.create",
+            "item": {
+                "type": "message",
+                "role": "user",
+                "content": [{"type": "input_text", "text": user_message}]
+            }
+        }))
+        
+        # Request response
+        await ws.send(json.dumps({
+            "type": "response.create",
+            "response": {"modalities": ["text", "audio"]}
+        }))
+        
+        # Stream response
+        print("🎤 Response: ", end='', flush=True)
+        transcript = []
+        
+        try:
+            while True:
+                msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
+                event = json.loads(msg)
+                
+                # Capture transcript deltas
+                if event['type'] == 'response.output_audio_transcript.delta':
+                    delta = event.get('delta', '')
+                    if delta:
+                        print(delta, end='', flush=True)
+                        transcript.append(delta)
+                
+                # Done when response completes
+                elif event['type'] == 'response.done':
+                    break
+        
+        except asyncio.TimeoutError:
+            pass
+        
+        print("\n")
+        
+        if transcript:
+            print(f"✅ Complete response: {''.join(transcript)}")
+        
+        await ws.close()
+
+
+def main():
+    """Run the voice agent"""
+    print("=" * 70)
+    print("LiveKit xAI Voice Agent via LiteLLM Proxy")
+    print("=" * 70)
+    print()
+    
+    try:
+        asyncio.run(run_voice_agent())
+    except KeyboardInterrupt:
+        print("\n\n👋 Goodbye!")
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        print("\nMake sure LiteLLM proxy is running:")
+        print(f"  litellm --config config.yaml --port 4000")
+
+
+if __name__ == "__main__":
+    main()

cookbook/livekit_agent_sdk/requirements.txt

@@ -0,0 +1,2 @@
+livekit-agents[xai]>=1.3.12
+websockets>=15.0.1

cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md

@@ -43,6 +43,14 @@ hide_table_of_contents: false
 ## Key Highlights
 [3-5 bullet points of major features - prioritize MCP OAuth 2.0, scheduled key rotations, and major model updates]
 
+## New Providers and Endpoints
+
+### New Providers
+[Table with Provider, Supported Endpoints, Description columns]
+
+### New LLM API Endpoints
+[Optional table for new endpoint additions with Endpoint, Method, Description, Documentation columns]
+
 ## New Models / Updated Models
 #### New Model Support
 [Model pricing table]
@@ -53,9 +61,6 @@ hide_table_of_contents: false
 ### Bug Fixes
 [Provider-specific bug fixes organized by provider]
 
-#### New Provider Support
-[New provider integrations]
-
 ## LLM API Endpoints
 #### Features
 [API-specific features organized by API type]
@@ -70,16 +75,20 @@ hide_table_of_contents: false
 #### Bugs
 [Management-related bug fixes]
 
-## Logging / Guardrail / Prompt Management Integrations
-#### Features
-[Organized by integration provider with proper doc links]
+## AI Integrations
 
-#### Guardrails
+### Logging
+[Logging integrations organized by provider with proper doc links, includes General subsection]
+
+### Guardrails
 [Guardrail-specific features and fixes]
 
-#### Prompt Management
+### Prompt Management
 [Prompt management integrations like BitBucket]
 
+### Secret Managers
+[Secret manager integrations - AWS, HashiCorp Vault, CyberArk, etc.]
+
 ## Spend Tracking, Budgets and Rate Limiting
 [Cost tracking, service tier pricing, rate limiting improvements]
 
@@ -149,26 +158,34 @@ hide_table_of_contents: false
 - Admin settings updates
 - Management routes and endpoints
 
-**Logging / Guardrail / Prompt Management Integrations:**
+**AI Integrations:**
 - **Structure:**
-  - `#### Features` - organized by integration provider with proper doc links
-  - `#### Guardrails` - guardrail-specific features and fixes
-  - `#### Prompt Management` - prompt management integrations
-  - `#### New Integration` - major new integrations
-- **Integration Categories:**
+  - `### Logging` - organized by integration provider with proper doc links, includes **General** subsection
+  - `### Guardrails` - guardrail-specific features and fixes
+  - `### Prompt Management` - prompt management integrations
+  - `### Secret Managers` - secret manager integrations
+- **Logging Categories:**
   - **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes
   - **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features
   - **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements
   - **[PostHog](../../docs/observability/posthog)** - observability integration
   - **[SQS](../../docs/proxy/logging#sqs)** - SQS logging features
   - **[Opik](../../docs/proxy/logging#opik)** - Opik integration improvements
+  - **[Arize Phoenix](../../docs/observability/arize_phoenix)** - Arize Phoenix integration
+  - **General** - miscellaneous logging features like callback controls, sensitive data masking
   - Other logging providers with proper doc links
 - **Guardrail Categories:**
-  - LakeraAI, Presidio, Noma, and other guardrail providers
+  - LakeraAI, Presidio, Noma, Grayswan, IBM Guardrails, and other guardrail providers
 - **Prompt Management:**
   - BitBucket, GitHub, and other prompt management integrations
+  - Prompt versioning, testing, and UI features
+- **Secret Managers:**
+  - **[AWS Secrets Manager](../../docs/secret_managers)** - AWS secret manager features
+  - **[HashiCorp Vault](../../docs/secret_managers)** - Vault integrations
+  - **[CyberArk](../../docs/secret_managers)** - CyberArk integrations
+  - **General** - cross-secret-manager features
 - Use bullet points under each provider for multiple features
-- Separate logging features from guardrails and prompt management clearly
+- Separate logging, guardrails, prompt management, and secret managers clearly
 
 ### 4. Documentation Linking Strategy
 
@@ -232,6 +249,9 @@ From git diff analysis, create tables like:
 - **Cost breakdown in logging** → Spend Tracking section
 - **MCP configuration/OAuth** → MCP Gateway (NOT General Proxy Improvements)
 - **All documentation PRs** → Documentation Updates section for visibility
+- **Callback controls/logging features** → AI Integrations > Logging > General
+- **Secret manager features** → AI Integrations > Secret Managers
+- **Video generation tag-based routing** → LLM API Endpoints > Video Generation API
 
 ### 7. Writing Style Guidelines
 
@@ -370,10 +390,107 @@ This release has a known issue...
   - **Virtual Keys** - Key rotation and management
   - **Models + Endpoints** - Provider and endpoint management
 
-**Logging Section Expansion:**
-- Rename to "Logging / Guardrail / Prompt Management Integrations"
-- Add **Prompt Management** subsection for BitBucket, GitHub integrations
-- Keep guardrails separate from logging features
+**AI Integrations Section Expansion:**
+- Renamed from "Logging / Guardrail / Prompt Management Integrations" to "AI Integrations"
+- Structure with four main subsections:
+  - **Logging** - with **General** subsection for miscellaneous logging features
+  - **Guardrails** - separate from logging features
+  - **Prompt Management** - BitBucket, GitHub integrations, versioning features
+  - **Secret Managers** - AWS, HashiCorp Vault, CyberArk, etc.
+
+**New Providers and Endpoints Section:**
+- Add section after Key Highlights and before New Models / Updated Models
+- Include tables for:
+  - **New Providers** - Provider name, supported endpoints, description
+  - **New LLM API Endpoints** (optional) - Endpoint, method, description, documentation link
+- Only include major new provider integrations, not minor provider updates
+- **IMPORTANT**: When adding new providers, also update `provider_endpoints_support.json` in the repository root (see Section 13)
+
+### 12. Section Header Counts
+
+**Always include counts in section headers for:**
+- **New Providers** - Add count in parentheses: `### New Providers (X new providers)`
+- **New LLM API Endpoints** - Add count in parentheses: `### New LLM API Endpoints (X new endpoints)`
+- **New Model Support** - Add count in parentheses: `#### New Model Support (X new models)`
+
+**Format:**
+```markdown
+### New Providers (4 new providers)
+
+| Provider | Supported LiteLLM Endpoints | Description |
+| -------- | --------------------------- | ----------- |
+...
+
+### New LLM API Endpoints (2 new endpoints)
+
+| Endpoint | Method | Description | Documentation |
+| -------- | ------ | ----------- | ------------- |
+...
+
+#### New Model Support (32 new models)
+
+| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
+| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
+...
+```
+
+**Counting Rules:**
+- Count each row in the table (excluding the header row)
+- For models, count each model entry in the pricing table
+- For providers, count each new provider added
+- For endpoints, count each new API endpoint added
+
+### 13. Update provider_endpoints_support.json
+
+**When adding new providers or endpoints, you MUST also update `provider_endpoints_support.json` in the repository root.**
+
+This file tracks which endpoints are supported by each LiteLLM provider and is used to generate documentation.
+
+**Required Steps:**
+1. For each new provider added to the release notes, add a corresponding entry to `provider_endpoints_support.json`
+2. For each new endpoint type added, update the schema comment and add the endpoint to relevant providers
+
+**Provider Entry Format:**
+```json
+"provider_slug": {
+  "display_name": "Provider Name (`provider_slug`)",
+  "url": "https://docs.litellm.ai/docs/providers/provider_slug",
+  "endpoints": {
+    "chat_completions": true,
+    "messages": true,
+    "responses": true,
+    "embeddings": false,
+    "image_generations": false,
+    "audio_transcriptions": false,
+    "audio_speech": false,
+    "moderations": false,
+    "batches": false,
+    "rerank": false,
+    "a2a": true
+  }
+}
+```
+
+**Available Endpoint Types:**
+- `chat_completions` - `/chat/completions` endpoint
+- `messages` - `/messages` endpoint (Anthropic format)
+- `responses` - `/responses` endpoint (OpenAI/Anthropic unified)
+- `embeddings` - `/embeddings` endpoint
+- `image_generations` - `/image/generations` endpoint
+- `audio_transcriptions` - `/audio/transcriptions` endpoint
+- `audio_speech` - `/audio/speech` endpoint
+- `moderations` - `/moderations` endpoint
+- `batches` - `/batches` endpoint
+- `rerank` - `/rerank` endpoint
+- `ocr` - `/ocr` endpoint
+- `search` - `/search` endpoint
+- `vector_stores` - `/vector_stores` endpoint
+- `a2a` - `/a2a/{agent}/message/send` endpoint (A2A Protocol)
+
+**Checklist:**
+- [ ] All new providers from release notes are added to `provider_endpoints_support.json`
+- [ ] Endpoint support flags accurately reflect provider capabilities
+- [ ] Documentation URL points to correct provider docs page
 
 ## Example Command Workflow
 

cookbook/mock_guardrail_server/mock_bedrock_guardrail_server.py

@@ -0,0 +1,540 @@
+#!/usr/bin/env python3
+"""
+Mock Bedrock Guardrail API Server
+
+This is a FastAPI server that mimics the AWS Bedrock Guardrail API for testing purposes.
+It follows the same API spec as the real Bedrock guardrail endpoint.
+
+Usage:
+    python mock_bedrock_guardrail_server.py
+
+The server will start on http://localhost:8080
+"""
+
+import os
+import re
+from typing import Any, Dict, List, Literal, Optional
+
+from fastapi import Depends, FastAPI, Header, HTTPException, status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+# ============================================================================
+# Request/Response Models (matching Bedrock API spec)
+# ============================================================================
+
+
+class BedrockTextContent(BaseModel):
+    text: str
+
+
+class BedrockContentItem(BaseModel):
+    text: BedrockTextContent
+
+
+class BedrockRequest(BaseModel):
+    source: Literal["INPUT", "OUTPUT"]
+    content: List[BedrockContentItem] = Field(default_factory=list)
+
+
+class BedrockGuardrailOutput(BaseModel):
+    text: Optional[str] = None
+
+
+class TopicPolicyItem(BaseModel):
+    name: str
+    type: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class TopicPolicy(BaseModel):
+    topics: List[TopicPolicyItem] = Field(default_factory=list)
+
+
+class ContentFilterItem(BaseModel):
+    type: str
+    confidence: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class ContentPolicy(BaseModel):
+    filters: List[ContentFilterItem] = Field(default_factory=list)
+
+
+class CustomWord(BaseModel):
+    match: str
+    action: Literal["BLOCKED", "NONE"]
+
+
+class WordPolicy(BaseModel):
+    customWords: List[CustomWord] = Field(default_factory=list)
+    managedWordLists: List[Dict[str, Any]] = Field(default_factory=list)
+
+
+class PiiEntity(BaseModel):
+    type: str
+    match: str
+    action: Literal["BLOCKED", "ANONYMIZED", "NONE"]
+
+
+class RegexMatch(BaseModel):
+    name: str
+    match: str
+    regex: str
+    action: Literal["BLOCKED", "ANONYMIZED", "NONE"]
+
+
+class SensitiveInformationPolicy(BaseModel):
+    piiEntities: List[PiiEntity] = Field(default_factory=list)
+    regexes: List[RegexMatch] = Field(default_factory=list)
+
+
+class ContextualGroundingFilter(BaseModel):
+    type: str
+    threshold: float
+    score: float
+    action: Literal["BLOCKED", "NONE"]
+
+
+class ContextualGroundingPolicy(BaseModel):
+    filters: List[ContextualGroundingFilter] = Field(default_factory=list)
+
+
+class Assessment(BaseModel):
+    topicPolicy: Optional[TopicPolicy] = None
+    contentPolicy: Optional[ContentPolicy] = None
+    wordPolicy: Optional[WordPolicy] = None
+    sensitiveInformationPolicy: Optional[SensitiveInformationPolicy] = None
+    contextualGroundingPolicy: Optional[ContextualGroundingPolicy] = None
+
+
+class BedrockGuardrailResponse(BaseModel):
+    usage: Dict[str, int] = Field(
+        default_factory=lambda: {"topicPolicyUnits": 1, "contentPolicyUnits": 1}
+    )
+    action: Literal["NONE", "GUARDRAIL_INTERVENED"] = "NONE"
+    outputs: List[BedrockGuardrailOutput] = Field(default_factory=list)
+    assessments: List[Assessment] = Field(default_factory=list)
+
+
+# ============================================================================
+# Mock Guardrail Configuration
+# ============================================================================
+
+
+class GuardrailConfig(BaseModel):
+    """Configuration for mock guardrail behavior"""
+
+    blocked_words: List[str] = Field(
+        default_factory=lambda: ["offensive", "inappropriate", "badword"]
+    )
+    blocked_topics: List[str] = Field(default_factory=lambda: ["violence", "illegal"])
+    pii_patterns: Dict[str, str] = Field(
+        default_factory=lambda: {
+            "EMAIL": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",
+            "PHONE": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b",
+            "SSN": r"\b\d{3}-\d{2}-\d{4}\b",
+            "CREDIT_CARD": r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b",
+        }
+    )
+    anonymize_pii: bool = True  # If True, ANONYMIZE PII; if False, BLOCK it
+    bearer_token: str = "mock-bedrock-token-12345"
+
+
+# Global config
+GUARDRAIL_CONFIG = GuardrailConfig()
+
+# ============================================================================
+# FastAPI App Setup
+# ============================================================================
+
+app = FastAPI(
+    title="Mock Bedrock Guardrail API",
+    description="Mock server mimicking AWS Bedrock Guardrail API",
+    version="1.0.0",
+)
+
+
+# ============================================================================
+# Authentication
+# ============================================================================
+
+
+async def verify_bearer_token(authorization: Optional[str] = Header(None)) -> str:
+    """
+    Verify the Bearer token from the Authorization header.
+
+    Args:
+        authorization: The Authorization header value
+
+    Returns:
+        The token if valid
+
+    Raises:
+        HTTPException: If token is missing or invalid
+    """
+    if authorization is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Missing Authorization header",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    # Check if it's a Bearer token
+    parts = authorization.split()
+    print(f"parts: {parts}")
+    if len(parts) != 2 or parts[0].lower() != "bearer":
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid Authorization header format. Expected: Bearer <token>",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    token = parts[1]
+
+    # Verify token
+    if token != GUARDRAIL_CONFIG.bearer_token:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Invalid bearer token",
+        )
+
+    return token
+
+
+# ============================================================================
+# Guardrail Logic
+# ============================================================================
+
+
+def check_blocked_words(text: str) -> Optional[WordPolicy]:
+    """Check if text contains blocked words"""
+    found_words = []
+    text_lower = text.lower()
+
+    for word in GUARDRAIL_CONFIG.blocked_words:
+        if word.lower() in text_lower:
+            found_words.append(CustomWord(match=word, action="BLOCKED"))
+
+    if found_words:
+        return WordPolicy(customWords=found_words)
+    return None
+
+
+def check_blocked_topics(text: str) -> Optional[TopicPolicy]:
+    """Check if text contains blocked topics"""
+    found_topics = []
+    text_lower = text.lower()
+
+    for topic in GUARDRAIL_CONFIG.blocked_topics:
+        if topic.lower() in text_lower:
+            found_topics.append(
+                TopicPolicyItem(name=topic, type=topic.upper(), action="BLOCKED")
+            )
+
+    if found_topics:
+        return TopicPolicy(topics=found_topics)
+    return None
+
+
+def check_pii(text: str) -> tuple[Optional[SensitiveInformationPolicy], str]:
+    """
+    Check for PII in text and return policy + anonymized text
+
+    Returns:
+        Tuple of (SensitiveInformationPolicy or None, anonymized_text)
+    """
+    pii_entities = []
+    anonymized_text = text
+    action = "ANONYMIZED" if GUARDRAIL_CONFIG.anonymize_pii else "BLOCKED"
+
+    for pii_type, pattern in GUARDRAIL_CONFIG.pii_patterns.items():
+        try:
+            # Compile the regex pattern with a timeout to prevent ReDoS attacks
+            compiled_pattern = re.compile(pattern)
+            matches = compiled_pattern.finditer(text)
+            for match in matches:
+                matched_text = match.group()
+                pii_entities.append(
+                    PiiEntity(type=pii_type, match=matched_text, action=action)
+                )
+
+                # Anonymize the text if configured
+                if GUARDRAIL_CONFIG.anonymize_pii:
+                    anonymized_text = anonymized_text.replace(
+                        matched_text, f"[{pii_type}_REDACTED]"
+                    )
+        except re.error:
+            # Invalid regex pattern - skip it and log a warning
+            print(f"Warning: Invalid regex pattern for PII type {pii_type}: {pattern}")
+            continue
+
+    if pii_entities:
+        return SensitiveInformationPolicy(piiEntities=pii_entities), anonymized_text
+
+    return None, text
+
+
+def process_guardrail_request(
+    request: BedrockRequest,
+) -> tuple[BedrockGuardrailResponse, List[str]]:
+    """
+    Process a guardrail request and return the response.
+
+    Returns:
+        Tuple of (response, list of output texts)
+    """
+    all_text_content = []
+    output_texts = []
+
+    # Extract all text from content items
+    for content_item in request.content:
+        if content_item.text and content_item.text.text:
+            all_text_content.append(content_item.text.text)
+
+    # Combine all text for analysis
+    combined_text = " ".join(all_text_content)
+
+    # Initialize response
+    response = BedrockGuardrailResponse()
+    assessment = Assessment()
+    has_intervention = False
+
+    # Check for blocked words
+    word_policy = check_blocked_words(combined_text)
+    if word_policy:
+        assessment.wordPolicy = word_policy
+        has_intervention = True
+
+    # Check for blocked topics
+    topic_policy = check_blocked_topics(combined_text)
+    if topic_policy:
+        assessment.topicPolicy = topic_policy
+        has_intervention = True
+
+    # Check for PII
+    for text in all_text_content:
+        pii_policy, anonymized_text = check_pii(text)
+        if pii_policy:
+            assessment.sensitiveInformationPolicy = pii_policy
+            if GUARDRAIL_CONFIG.anonymize_pii:
+                # If anonymizing, we don't block, we modify the text
+                output_texts.append(anonymized_text)
+                has_intervention = True
+            else:
+                # If not anonymizing PII, we block it
+                output_texts.append(text)
+                has_intervention = True
+        else:
+            output_texts.append(text)
+
+    # Build response
+    if has_intervention:
+        response.action = "GUARDRAIL_INTERVENED"
+        # Only add assessment if there were interventions
+        response.assessments = [assessment]
+
+    # Add outputs (modified or original text)
+    response.outputs = [BedrockGuardrailOutput(text=txt) for txt in output_texts]
+
+    return response, output_texts
+
+
+# ============================================================================
+# API Endpoints
+# ============================================================================
+
+
+@app.get("/")
+async def root():
+    """Health check endpoint"""
+    return {
+        "service": "Mock Bedrock Guardrail API",
+        "status": "running",
+        "endpoint_format": "/guardrail/{guardrailIdentifier}/version/{guardrailVersion}/apply",
+    }
+
+
+@app.get("/health")
+async def health():
+    """Health check endpoint"""
+    return {"status": "healthy"}
+
+
+"""
+LiteLLM exposes a basic guardrail API with the text extracted from the request and sent to the guardrail API, as well as the received request body for any further processing. 
+
+This works across all LiteLLM endpoints (completion, anthropic /v1/messages, responses api, image generation, embedding, etc.)
+
+This makes it easy to support your own guardrail API without having to make a PR to LiteLLM.
+
+LiteLLM supports passing any provider specific params from LiteLLM config.yaml to the guardrail API.
+
+Example:
+
+```yaml
+guardrails:
+  - guardrail_name: "bedrock-content-guard"
+    litellm_params:
+      guardrail: generic_guardrail_api
+      mode: "pre_call"
+      api_key: os.environ/GUARDRAIL_API_KEY
+      api_base: os.environ/GUARDRAIL_API_BASE
+      additional_provider_specific_params:
+        api_version: os.environ/GUARDRAIL_API_VERSION # additional provider specific params
+```
+
+This is a beta API. Please help us improve it. 
+"""
+
+
+class LitellmBasicGuardrailRequest(BaseModel):
+    texts: List[str]
+    images: Optional[List[str]] = None
+    tools: Optional[List[dict]] = None
+    tool_calls: Optional[List[dict]] = None
+    request_data: Dict[str, Any] = Field(default_factory=dict)
+    additional_provider_specific_params: Dict[str, Any] = Field(default_factory=dict)
+    input_type: Literal["request", "response"]
+    litellm_call_id: Optional[str] = None
+    litellm_trace_id: Optional[str] = None
+    structured_messages: Optional[List[Dict[str, Any]]] = None
+
+
+class LitellmBasicGuardrailResponse(BaseModel):
+    action: Literal[
+        "BLOCKED", "NONE", "GUARDRAIL_INTERVENED"
+    ]  # BLOCKED = litellm will raise an error, NONE = litellm will continue, GUARDRAIL_INTERVENED = litellm will continue, but the text was modified by the guardrail
+    blocked_reason: Optional[str] = None  # only if action is BLOCKED, otherwise None
+    texts: Optional[List[str]] = None
+    images: Optional[List[str]] = None
+
+
+@app.post(
+    "/beta/litellm_basic_guardrail_api",
+    response_model=LitellmBasicGuardrailResponse,
+)
+async def beta_litellm_basic_guardrail_api(
+    request: LitellmBasicGuardrailRequest,
+) -> LitellmBasicGuardrailResponse:
+    """
+    Apply guardrail to input or output content.
+
+    This endpoint mimics the AWS Bedrock ApplyGuardrail API.
+
+    Args:
+        request: The guardrail request containing content to analyze
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        LitellmBasicGuardrailResponse with analysis results
+    """
+    print(f"request: {request}")
+    if any("ishaan" in text.lower() for text in request.texts):
+        return LitellmBasicGuardrailResponse(
+            action="BLOCKED", blocked_reason="Ishaan is not allowed"
+        )
+    elif any("pii_value" in text for text in request.texts):
+        return LitellmBasicGuardrailResponse(
+            action="GUARDRAIL_INTERVENED",
+            texts=[
+                text.replace("pii_value", "pii_value_redacted")
+                for text in request.texts
+            ],
+        )
+    return LitellmBasicGuardrailResponse(action="NONE")
+
+
+@app.post("/config/update")
+async def update_config(
+    config: GuardrailConfig, token: str = Depends(verify_bearer_token)
+):
+    """
+    Update the guardrail configuration.
+
+    This is a testing endpoint to modify the mock guardrail behavior.
+
+    Args:
+        config: New guardrail configuration
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        Updated configuration
+    """
+    global GUARDRAIL_CONFIG
+    GUARDRAIL_CONFIG = config
+    return {"status": "updated", "config": GUARDRAIL_CONFIG}
+
+
+@app.get("/config")
+async def get_config(token: str = Depends(verify_bearer_token)):
+    """
+    Get the current guardrail configuration.
+
+    Args:
+        token: Bearer token (verified by dependency)
+
+    Returns:
+        Current configuration
+    """
+    return GUARDRAIL_CONFIG
+
+
+# ============================================================================
+# Error Handlers
+# ============================================================================
+
+
+@app.exception_handler(HTTPException)
+async def http_exception_handler(request, exc: HTTPException):
+    """Custom error handler for HTTP exceptions"""
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={"error": exc.detail},
+        headers=exc.headers,
+    )
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+if __name__ == "__main__":
+    import uvicorn
+
+    # Get configuration from environment
+    host = os.getenv("MOCK_BEDROCK_HOST", "0.0.0.0")
+    port = int(os.getenv("MOCK_BEDROCK_PORT", "8080"))
+    bearer_token = os.getenv("MOCK_BEDROCK_TOKEN", "mock-bedrock-token-12345")
+
+    # Update config with environment token
+    GUARDRAIL_CONFIG.bearer_token = bearer_token
+
+    print("=" * 80)
+    print("Mock Bedrock Guardrail API Server")
+    print("=" * 80)
+    print(f"Server starting on: http://{host}:{port}")
+    print(f"Bearer Token: {bearer_token}")
+    print(f"Endpoint: POST /guardrail/{{id}}/version/{{version}}/apply")
+    print("=" * 80)
+    print("\nExample curl command:")
+    print(
+        f"""
+curl -X POST "http://{host}:{port}/guardrail/test-guardrail/version/1/apply" \\
+  -H "Authorization: Bearer {bearer_token}" \\
+  -H "Content-Type: application/json" \\
+  -d '{{
+    "source": "INPUT",
+    "content": [
+      {{
+        "text": {{
+          "text": "Hello, my email is test@example.com"
+        }}
+      }}
+    ]
+  }}'
+    """
+    )
+    print("=" * 80)
+
+    uvicorn.run(app, host=host, port=port)

cookbook/mock_prompt_management_server/README.md

@@ -0,0 +1,293 @@
+# Mock Prompt Management Server
+
+A reference implementation of the [LiteLLM Generic Prompt Management API](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api).
+
+This FastAPI server demonstrates how to build a prompt management API that integrates with LiteLLM without requiring a PR to the LiteLLM repository.
+
+## Quick Start
+
+### 1. Install Dependencies
+
+```bash
+pip install fastapi uvicorn pydantic
+```
+
+### 2. Start the Server
+
+```bash
+python mock_prompt_management_server.py
+```
+
+The server will start on `http://localhost:8080`
+
+### 3. Test the Endpoint
+
+```bash
+# Get a prompt
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
+
+# Get a prompt with authentication
+curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt" \
+  -H "Authorization: Bearer test-token-12345"
+
+# List all prompts
+curl "http://localhost:8080/prompts"
+
+# Get prompt variables
+curl "http://localhost:8080/prompts/hello-world-prompt/variables"
+```
+
+## Using with LiteLLM
+
+### Configuration
+
+Create a `config.yaml` file:
+
+```yaml
+model_list:
+  - model_name: gpt-3.5-turbo
+    litellm_params:
+      model: openai/gpt-3.5-turbo
+      api_key: os.environ/OPENAI_API_KEY
+
+prompts:
+  - prompt_id: "hello-world-prompt"
+    litellm_params:
+      prompt_integration: "generic_prompt_management"
+      api_base: http://localhost:8080
+      api_key: test-token-12345
+```
+
+### Start LiteLLM Proxy
+
+```bash
+litellm --config config.yaml
+```
+
+### Make a Request
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer sk-1234" \
+  -d '{
+    "model": "gpt-3.5-turbo",
+    "prompt_id": "hello-world-prompt",
+    "prompt_variables": {
+      "domain": "data science",
+      "task": "analyzing customer behavior"
+    },
+    "messages": [
+      {"role": "user", "content": "Please help me get started"}
+    ]
+  }'
+```
+
+## Available Prompts
+
+The server includes several example prompts:
+
+| Prompt ID | Description | Variables |
+|-----------|-------------|-----------|
+| `hello-world-prompt` | Basic helpful assistant | `domain`, `task` |
+| `code-review-prompt` | Code review assistant | `years_experience`, `language`, `code` |
+| `customer-support-prompt` | Customer support agent | `company_name`, `customer_message` |
+| `data-analysis-prompt` | Data analysis expert | `analysis_type`, `dataset_name`, `data` |
+| `creative-writing-prompt` | Creative writing assistant | `genre`, `length`, `topic` |
+
+## Authentication
+
+The server supports optional Bearer token authentication. Valid tokens for testing:
+
+- `test-token-12345`
+- `dev-token-67890`
+- `prod-token-abcdef`
+
+If no `Authorization` header is provided, requests are allowed (for testing purposes).
+
+## API Endpoints
+
+### LiteLLM Spec Endpoints
+
+#### `GET /beta/litellm_prompt_management`
+
+Get a prompt by ID (required by LiteLLM).
+
+**Query Parameters:**
+- `prompt_id` (required): The prompt ID
+- `project_name` (optional): Project filter
+- `slug` (optional): Slug filter
+- `version` (optional): Version filter
+
+**Response:**
+```json
+{
+  "prompt_id": "hello-world-prompt",
+  "prompt_template": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant specialized in {domain}."
+    },
+    {
+      "role": "user",
+      "content": "Help me with: {task}"
+    }
+  ],
+  "prompt_template_model": "gpt-4",
+  "prompt_template_optional_params": {
+    "temperature": 0.7,
+    "max_tokens": 500
+  }
+}
+```
+
+### Convenience Endpoints (Not in LiteLLM Spec)
+
+#### `GET /health`
+
+Health check endpoint.
+
+#### `GET /prompts`
+
+List all available prompts.
+
+#### `GET /prompts/{prompt_id}/variables`
+
+Get all variables used in a prompt template.
+
+#### `POST /prompts`
+
+Create a new prompt (in-memory only, for testing).
+
+## Example: Full Integration Test
+
+### 1. Start the Mock Server
+
+```bash
+python mock_prompt_management_server.py
+```
+
+### 2. Test with Python
+
+```python
+from litellm import completion
+
+# The completion will:
+# 1. Fetch the prompt from your API
+# 2. Replace {domain} with "machine learning"
+# 3. Replace {task} with "building a recommendation system"
+# 4. Merge with your messages
+# 5. Use the model and params from the prompt
+
+response = completion(
+    model="gpt-4",
+    prompt_id="hello-world-prompt",
+    prompt_variables={
+        "domain": "machine learning",
+        "task": "building a recommendation system"
+    },
+    messages=[
+        {"role": "user", "content": "I have user behavior data from the past year."}
+    ],
+    # Configure the generic prompt manager
+    generic_prompt_config={
+        "api_base": "http://localhost:8080",
+        "api_key": "test-token-12345",
+    }
+)
+
+print(response.choices[0].message.content)
+```
+
+## Customization
+
+### Adding New Prompts
+
+Edit the `PROMPTS_DB` dictionary in `mock_prompt_management_server.py`:
+
+```python
+PROMPTS_DB = {
+    "my-custom-prompt": {
+        "prompt_id": "my-custom-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a {role}."
+            },
+            {
+                "role": "user",
+                "content": "{user_input}"
+            }
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.8,
+            "max_tokens": 1000
+        }
+    }
+}
+```
+
+### Using a Database
+
+Replace the `PROMPTS_DB` dictionary with database queries:
+
+```python
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(prompt_id: str):
+    # Fetch from database
+    prompt = await db.prompts.find_one({"prompt_id": prompt_id})
+    
+    if not prompt:
+        raise HTTPException(status_code=404, detail="Prompt not found")
+    
+    return PromptResponse(**prompt)
+```
+
+### Adding Access Control
+
+Use the custom query parameters for access control:
+
+```python
+@app.get("/beta/litellm_prompt_management")
+async def get_prompt(
+    prompt_id: str,
+    project_name: Optional[str] = None,
+    user_id: Optional[str] = None,
+    authorization: Optional[str] = Header(None)
+):
+    token = verify_api_key(authorization)
+    
+    # Check if user has access to this project
+    if not has_project_access(token, project_name):
+        raise HTTPException(status_code=403, detail="Access denied")
+    
+    # Fetch and return prompt
+    ...
+```
+
+## Production Considerations
+
+Before deploying to production:
+
+1. **Use a real database** instead of in-memory storage
+2. **Implement proper authentication** with JWT tokens or API keys
+3. **Add rate limiting** to prevent abuse
+4. **Use HTTPS** for encrypted communication
+5. **Add logging and monitoring** for observability
+6. **Implement caching** for frequently accessed prompts
+7. **Add versioning** for prompt management
+8. **Implement access control** based on teams/users
+9. **Add input validation** for all parameters
+10. **Use environment variables** for configuration
+
+## Related Documentation
+
+- [Generic Prompt Management API Documentation](https://docs.litellm.ai/docs/adding_provider/generic_prompt_management_api)
+- [LiteLLM Prompt Management](https://docs.litellm.ai/docs/proxy/prompt_management)
+- [Generic Guardrail API](https://docs.litellm.ai/docs/adding_provider/generic_guardrail_api)
+
+## Questions?
+
+This is a reference implementation for the LiteLLM Generic Prompt Management API. For questions or issues, please open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm).
+

cookbook/mock_prompt_management_server/mock_prompt_management_server.py

@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+"""
+Mock Prompt Management API Server
+
+This is a FastAPI server that implements the LiteLLM Generic Prompt Management API
+for testing and demonstration purposes.
+
+Usage:
+    python mock_prompt_management_server.py
+
+The server will start on http://localhost:8080
+
+Test the endpoint:
+    curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"
+"""
+
+import os
+import json
+from typing import Any, Dict, List, Optional
+
+from fastapi import FastAPI, HTTPException, Header, Query, status
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+# ============================================================================
+# Response Models
+# ============================================================================
+
+
+class MessageContent(BaseModel):
+    """A single message in the prompt template"""
+
+    role: str = Field(..., description="Message role (system, user, assistant)")
+    content: str = Field(
+        ..., description="Message content with optional {variable} placeholders"
+    )
+
+
+class PromptResponse(BaseModel):
+    """Response format for the prompt management API"""
+
+    prompt_id: str = Field(..., description="The ID of the prompt")
+    prompt_template: List[MessageContent] = Field(
+        ..., description="Array of messages in OpenAI format"
+    )
+    prompt_template_model: Optional[str] = Field(
+        None, description="Optional model to use for this prompt"
+    )
+    prompt_template_optional_params: Optional[Dict[str, Any]] = Field(
+        None, description="Optional parameters like temperature, max_tokens, etc."
+    )
+
+
+# ============================================================================
+# Mock Prompt Database
+# ============================================================================
+
+PROMPTS_DB = {
+    "hello-world-prompt": {
+        "prompt_id": "hello-world-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a helpful assistant specialized in {domain}.",
+            },
+            {"role": "user", "content": "Help me with: {task}"},
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {"temperature": 0.7, "max_tokens": 500},
+    },
+    "code-review-prompt": {
+        "prompt_id": "code-review-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are an expert code reviewer with {years_experience} years of experience in {language}.",
+            },
+            {
+                "role": "user",
+                "content": "Please review the following code for bugs, security issues, and best practices:\n\n{code}",
+            },
+        ],
+        "prompt_template_model": "gpt-4-turbo",
+        "prompt_template_optional_params": {
+            "temperature": 0.3,
+            "max_tokens": 1500,
+        },
+    },
+    "customer-support-prompt": {
+        "prompt_id": "customer-support-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a friendly customer support agent for {company_name}. Always be professional, empathetic, and solution-oriented.",
+            },
+            {
+                "role": "user",
+                "content": "Customer inquiry: {customer_message}",
+            },
+        ],
+        "prompt_template_model": "gpt-3.5-turbo",
+        "prompt_template_optional_params": {
+            "temperature": 0.8,
+            "max_tokens": 800,
+            "top_p": 0.9,
+        },
+    },
+    "data-analysis-prompt": {
+        "prompt_id": "data-analysis-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a data scientist expert in {analysis_type} analysis.",
+            },
+            {
+                "role": "user",
+                "content": "Analyze the following data and provide insights:\n\nDataset: {dataset_name}\nData: {data}",
+            },
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.5,
+            "max_tokens": 2000,
+        },
+    },
+    "creative-writing-prompt": {
+        "prompt_id": "creative-writing-prompt",
+        "prompt_template": [
+            {
+                "role": "system",
+                "content": "You are a creative writer specializing in {genre} fiction.",
+            },
+            {
+                "role": "user",
+                "content": "Write a {length} story about: {topic}",
+            },
+        ],
+        "prompt_template_model": "gpt-4",
+        "prompt_template_optional_params": {
+            "temperature": 0.9,
+            "max_tokens": 3000,
+            "top_p": 0.95,
+        },
+    },
+}
+
+# Valid API tokens for authentication (in production, use a secure token store)
+VALID_API_TOKENS = {
+    "test-token-12345",
+    "dev-token-67890",
+    "prod-token-abcdef",
+}
+
+# ============================================================================
+# FastAPI App
+# ============================================================================
+
+app = FastAPI(
+    title="Mock Prompt Management API",
+    description="A mock server implementing the LiteLLM Generic Prompt Management API",
+    version="1.0.0",
+)
+
+
+def verify_api_key(authorization: Optional[str] = Header(None)) -> bool:
+    """
+    Verify the API key from the Authorization header.
+
+    Args:
+        authorization: Authorization header (Bearer token)
+
+    Returns:
+        True if valid, raises HTTPException if invalid
+    """
+    if authorization is None:
+        # Allow requests without authentication for testing
+        return True
+
+    # Extract token from "Bearer <token>"
+    if not authorization.startswith("Bearer "):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid authorization header format. Expected 'Bearer <token>'",
+        )
+
+    token = authorization.replace("Bearer ", "").strip()
+
+    if token not in VALID_API_TOKENS:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid API key",
+        )
+
+    return True
+
+
+@app.get("/beta/litellm_prompt_management", response_model=PromptResponse)
+async def get_prompt(
+    prompt_id: str = Query(..., description="The ID of the prompt to fetch"),
+    project_name: Optional[str] = Query(
+        None, description="Optional project name filter"
+    ),
+    slug: Optional[str] = Query(None, description="Optional slug filter"),
+    version: Optional[str] = Query(None, description="Optional version filter"),
+    authorization: Optional[str] = Header(None),
+) -> PromptResponse:
+    """
+    Get a prompt by ID with optional filtering.
+
+    This endpoint implements the LiteLLM Generic Prompt Management API specification.
+
+    Args:
+        prompt_id: The ID of the prompt to fetch
+        project_name: Optional project name for filtering
+        slug: Optional slug for filtering
+        version: Optional version for filtering
+        authorization: Optional Bearer token for authentication
+
+    Returns:
+        PromptResponse with the prompt template and configuration
+
+    Raises:
+        HTTPException: 401 if authentication fails, 404 if prompt not found
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    # Log the request parameters (useful for debugging)
+    print(f"Fetching prompt: {prompt_id}")
+    if project_name:
+        print(f"  Project: {project_name}")
+    if slug:
+        print(f"  Slug: {slug}")
+    if version:
+        print(f"  Version: {version}")
+
+    # Check if prompt exists
+    if prompt_id not in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Prompt '{prompt_id}' not found. Available prompts: {list(PROMPTS_DB.keys())}",
+        )
+
+    # Get the prompt from the database
+    prompt_data = PROMPTS_DB[prompt_id]
+
+    # Optional: Apply filtering based on project_name, slug, or version
+    # In a real implementation, you might use these to filter prompts by access control
+    # or to fetch specific versions from your database
+
+    return PromptResponse(**prompt_data)
+
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint"""
+    return {
+        "status": "healthy",
+        "service": "mock-prompt-management-api",
+        "version": "1.0.0",
+    }
+
+
+@app.get("/prompts")
+async def list_prompts(authorization: Optional[str] = Header(None)):
+    """
+    List all available prompts.
+
+    This is a convenience endpoint (not part of the LiteLLM spec) for
+    discovering available prompts.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    prompts_list = [
+        {
+            "prompt_id": pid,
+            "model": p.get("prompt_template_model"),
+            "has_variables": any(
+                "{" in msg.get("content", "") for msg in p.get("prompt_template", [])
+            ),
+        }
+        for pid, p in PROMPTS_DB.items()
+    ]
+
+    return {"prompts": prompts_list, "total": len(prompts_list)}
+
+
+@app.get("/prompts/{prompt_id}/variables")
+async def get_prompt_variables(
+    prompt_id: str, authorization: Optional[str] = Header(None)
+):
+    """
+    Get all variables in a prompt template.
+
+    This is a convenience endpoint (not part of the LiteLLM spec) for
+    discovering what variables a prompt expects.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    if prompt_id not in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Prompt '{prompt_id}' not found",
+        )
+
+    prompt_data = PROMPTS_DB[prompt_id]
+    variables = set()
+
+    # Extract variables from the prompt template
+    import re
+
+    for message in prompt_data["prompt_template"]:
+        content = message.get("content", "")
+        # Find all {variable} patterns
+        found_vars = re.findall(r"\{(\w+)\}", content)
+        variables.update(found_vars)
+
+    return {
+        "prompt_id": prompt_id,
+        "variables": sorted(list(variables)),
+        "example_usage": {
+            "prompt_id": prompt_id,
+            "prompt_variables": {var: f"<{var}_value>" for var in variables},
+        },
+    }
+
+
+@app.post("/prompts")
+async def create_prompt(
+    prompt: PromptResponse, authorization: Optional[str] = Header(None)
+):
+    """
+    Create a new prompt (convenience endpoint for testing).
+
+    This is NOT part of the LiteLLM spec - it's just for testing purposes.
+    """
+    # Verify authentication
+    verify_api_key(authorization)
+
+    if prompt.prompt_id in PROMPTS_DB:
+        raise HTTPException(
+            status_code=status.HTTP_409_CONFLICT,
+            detail=f"Prompt '{prompt.prompt_id}' already exists",
+        )
+
+    PROMPTS_DB[prompt.prompt_id] = prompt.dict()
+
+    return {
+        "status": "created",
+        "prompt_id": prompt.prompt_id,
+        "message": "Prompt created successfully (in-memory only)",
+    }
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+if __name__ == "__main__":
+    import uvicorn
+
+    print("=" * 70)
+    print("Mock Prompt Management API Server")
+    print("=" * 70)
+    print(f"\nStarting server on http://localhost:8080")
+    print(f"\nAvailable prompts: {len(PROMPTS_DB)}")
+    for prompt_id in PROMPTS_DB.keys():
+        print(f"  - {prompt_id}")
+    print(f"\nValid API tokens: {len(VALID_API_TOKENS)}")
+    print("  - test-token-12345")
+    print("  - dev-token-67890")
+    print("  - prod-token-abcdef")
+    print("\nEndpoints:")
+    print("  GET  /beta/litellm_prompt_management?prompt_id=<id>  (LiteLLM spec)")
+    print("  GET  /health                                          (health check)")
+    print("  GET  /prompts                                         (list all prompts)")
+    print(
+        "  GET  /prompts/{id}/variables                          (get prompt variables)"
+    )
+    print("  POST /prompts                                         (create prompt)")
+    print("\nExample usage:")
+    print(
+        '  curl "http://localhost:8080/beta/litellm_prompt_management?prompt_id=hello-world-prompt"'
+    )
+    print("\nPress CTRL+C to stop the server")
+    print("=" * 70)
+
+    uvicorn.run(app, host="0.0.0.0", port=8080, log_level="info")

cookbook/nova_sonic_realtime.py

@@ -0,0 +1,288 @@
+"""
+Client script to test Nova Sonic realtime API through LiteLLM proxy.
+
+This script connects to LiteLLM proxy's realtime endpoint and enables
+speech-to-speech conversation with Bedrock Nova Sonic.
+
+Prerequisites:
+- LiteLLM proxy running with Bedrock configured
+- pyaudio installed: pip install pyaudio
+- websockets installed: pip install websockets
+
+Usage:
+    python nova_sonic_realtime.py
+"""
+
+import asyncio
+import base64
+import json
+import os
+import pyaudio
+import websockets
+from typing import Optional
+
+# Bounded queue size for audio chunks (configurable via env to avoid unbounded memory)
+AUDIO_QUEUE_MAXSIZE = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 10_000))
+
+# Audio configuration (matching Nova Sonic requirements)
+INPUT_SAMPLE_RATE = 16000  # Nova Sonic expects 16kHz input
+OUTPUT_SAMPLE_RATE = 24000  # Nova Sonic outputs 24kHz
+CHANNELS = 1
+FORMAT = pyaudio.paInt16
+CHUNK_SIZE = 1024
+
+# LiteLLM proxy configuration
+LITELLM_PROXY_URL = "ws://localhost:4000/v1/realtime?model=bedrock-sonic"
+LITELLM_API_KEY = "sk-12345"  # Your LiteLLM API key
+
+
+class RealtimeClient:
+    """Client for LiteLLM realtime API with audio support."""
+
+    def __init__(self, url: str, api_key: str):
+        self.url = url
+        self.api_key = api_key
+        self.ws: Optional[websockets.WebSocketClientProtocol] = None
+        self.is_active = False
+        self.audio_queue = asyncio.Queue(maxsize=AUDIO_QUEUE_MAXSIZE)
+        self.pyaudio = pyaudio.PyAudio()
+        self.input_stream = None
+        self.output_stream = None
+
+    async def connect(self):
+        """Connect to LiteLLM proxy realtime endpoint."""
+        print(f"Connecting to {self.url}...")
+        
+        headers = {}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        
+        self.ws = await websockets.connect(
+            self.url,
+            additional_headers=headers,
+            max_size=10 * 1024 * 1024,  # 10MB max message size
+        )
+        self.is_active = True
+        print("✓ Connected to LiteLLM proxy")
+
+    async def send_session_update(self):
+        """Send session configuration."""
+        session_update = {
+            "type": "session.update",
+            "session": {
+                "instructions": "You are a friendly assistant. Keep your responses short and conversational.",
+                "voice": "matthew",
+                "temperature": 0.8,
+                "max_response_output_tokens": 1024,
+                "modalities": ["text", "audio"],
+                "input_audio_format": "pcm16",
+                "output_audio_format": "pcm16",
+                "turn_detection": {
+                    "type": "server_vad",
+                    "threshold": 0.5,
+                    "prefix_padding_ms": 300,
+                    "silence_duration_ms": 500,
+                },
+            },
+        }
+        await self.ws.send(json.dumps(session_update))
+        print("✓ Session configuration sent")
+
+    async def receive_messages(self):
+        """Receive and process messages from the server."""
+        try:
+            async for message in self.ws:
+                if not self.is_active:
+                    break
+
+                try:
+                    data = json.loads(message)
+                    event_type = data.get("type")
+
+                    if event_type == "session.created":
+                        print(f"✓ Session created: {data.get('session', {}).get('id')}")
+
+                    elif event_type == "response.created":
+                        print("🤖 Assistant is responding...")
+
+                    elif event_type == "response.text.delta":
+                        # Print text transcription
+                        delta = data.get("delta", "")
+                        print(delta, end="", flush=True)
+
+                    elif event_type == "response.audio.delta":
+                        # Queue audio for playback
+                        audio_b64 = data.get("delta", "")
+                        if audio_b64:
+                            audio_bytes = base64.b64decode(audio_b64)
+                            await self.audio_queue.put(audio_bytes)
+
+                    elif event_type == "response.text.done":
+                        print()  # New line after text
+
+                    elif event_type == "response.done":
+                        print("✓ Response complete")
+
+                    elif event_type == "error":
+                        print(f"❌ Error: {data.get('error', {})}")
+
+                    else:
+                        # Debug: print other event types
+                        print(f"[{event_type}]", end=" ")
+
+                except json.JSONDecodeError:
+                    print(f"Failed to parse message: {message[:100]}")
+
+        except websockets.exceptions.ConnectionClosed:
+            print("\n✗ Connection closed")
+        except Exception as e:
+            print(f"\n✗ Error receiving messages: {e}")
+        finally:
+            self.is_active = False
+
+    async def send_audio_chunk(self, audio_bytes: bytes):
+        """Send audio chunk to server."""
+        if not self.is_active or not self.ws:
+            return
+
+        audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
+        message = {
+            "type": "input_audio_buffer.append",
+            "audio": audio_b64,
+        }
+        await self.ws.send(json.dumps(message))
+
+    async def commit_audio_buffer(self):
+        """Commit the audio buffer to trigger processing."""
+        if not self.is_active or not self.ws:
+            return
+
+        message = {"type": "input_audio_buffer.commit"}
+        await self.ws.send(json.dumps(message))
+
+    async def capture_audio(self):
+        """Capture audio from microphone and send to server."""
+        print("\n🎤 Starting audio capture...")
+        print("Speak into your microphone. Press Ctrl+C to stop.\n")
+
+        self.input_stream = self.pyaudio.open(
+            format=FORMAT,
+            channels=CHANNELS,
+            rate=INPUT_SAMPLE_RATE,
+            input=True,
+            frames_per_buffer=CHUNK_SIZE,
+        )
+
+        try:
+            while self.is_active:
+                audio_data = self.input_stream.read(CHUNK_SIZE, exception_on_overflow=False)
+                await self.send_audio_chunk(audio_data)
+                await asyncio.sleep(0.01)  # Small delay to prevent overwhelming
+        except Exception as e:
+            print(f"Error capturing audio: {e}")
+        finally:
+            if self.input_stream:
+                self.input_stream.stop_stream()
+                self.input_stream.close()
+
+    async def play_audio(self):
+        """Play audio responses from the server."""
+        print("🔊 Starting audio playback...")
+
+        self.output_stream = self.pyaudio.open(
+            format=FORMAT,
+            channels=CHANNELS,
+            rate=OUTPUT_SAMPLE_RATE,
+            output=True,
+            frames_per_buffer=CHUNK_SIZE,
+        )
+
+        try:
+            while self.is_active:
+                try:
+                    audio_data = await asyncio.wait_for(
+                        self.audio_queue.get(), timeout=0.1
+                    )
+                    if audio_data:
+                        self.output_stream.write(audio_data)
+                except asyncio.TimeoutError:
+                    continue
+        except Exception as e:
+            print(f"Error playing audio: {e}")
+        finally:
+            if self.output_stream:
+                self.output_stream.stop_stream()
+                self.output_stream.close()
+
+    async def close(self):
+        """Close the connection and cleanup."""
+        self.is_active = False
+
+        if self.ws:
+            await self.ws.close()
+
+        if self.input_stream:
+            self.input_stream.stop_stream()
+            self.input_stream.close()
+
+        if self.output_stream:
+            self.output_stream.stop_stream()
+            self.output_stream.close()
+
+        self.pyaudio.terminate()
+        print("\n✓ Connection closed")
+
+
+async def main():
+    """Main function to run the realtime client."""
+    print("=" * 80)
+    print("Bedrock Nova Sonic Realtime Client")
+    print("=" * 80)
+    print()
+
+    client = RealtimeClient(LITELLM_PROXY_URL, LITELLM_API_KEY)
+
+    try:
+        # Connect to server
+        await client.connect()
+
+        # Send session configuration
+        await client.send_session_update()
+
+        # Wait a moment for session to be established
+        await asyncio.sleep(0.5)
+
+        # Start tasks
+        receive_task = asyncio.create_task(client.receive_messages())
+        capture_task = asyncio.create_task(client.capture_audio())
+        playback_task = asyncio.create_task(client.play_audio())
+
+        # Wait for user to interrupt
+        await asyncio.gather(
+            receive_task,
+            capture_task,
+            playback_task,
+            return_exceptions=True,
+        )
+
+    except KeyboardInterrupt:
+        print("\n\n⚠ Interrupted by user")
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        import traceback
+        traceback.print_exc()
+    finally:
+        await client.close()
+
+
+if __name__ == "__main__":
+    print("\nMake sure:")
+    print("1. LiteLLM proxy is running on port 4000")
+    print("2. Bedrock is configured in proxy_server_config.yaml")
+    print("3. AWS credentials are set")
+    print()
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\n\nGoodbye!")

deploy/Dockerfile.ghcr_base

@@ -8,7 +8,8 @@ WORKDIR /app
 COPY config.yaml .
 
 # Make sure your docker/entrypoint.sh is executable
-RUN chmod +x docker/entrypoint.sh
+# Convert Windows line endings to Unix
+RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
 
 # Expose the necessary port
 EXPOSE 4000/tcp

deploy/charts/litellm-helm/Chart.yaml

@@ -18,13 +18,17 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.4.7
+version: 1.1.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: v1.50.2
+appVersion: v1.80.12
+
+annotations:
+  org.opencontainers.image.source: "https://github.com/BerriAI/litellm"
+  org.opencontainers.image.url: "https://docs.litellm.ai/"
 
 dependencies:
   - name: "postgresql"
@@ -33,5 +37,5 @@ dependencies:
     condition: db.deployStandalone
   - name: redis
     version: ">=18.0.0"
-    repository: oci://registry-1.docker.io/bitnamicharts 
+    repository: oci://registry-1.docker.io/bitnamicharts
     condition: redis.enabled

deploy/charts/litellm-helm/README.md

@@ -10,46 +10,52 @@
 - Helm 3.8.0+
 
 If `db.deployStandalone` is used:
+
 - PV provisioner support in the underlying infrastructure
 
 If `db.useStackgresOperator` is used (not yet implemented):
-- The Stackgres Operator must already be installed in the Kubernetes Cluster.  This chart will **not** install the operator if it is missing.
+
+- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing.
 
 ## Parameters
 
 ### LiteLLM Proxy Deployment Settings
 
-| Name                                                       | Description                                                                                                                                                                           | Value |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| `replicaCount`                                             | The number of LiteLLM Proxy pods to be deployed                                                                                                                                       | `1`  |
-| `masterkeySecretName`                                      | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM.  If not specified, use the generated secret name.                                                                                                         | N/A  |
-| `masterkeySecretKey`                                      | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM.  If not specified, use `masterkey` as the key.                                                                                                         | N/A  |
-| `masterkey`                                                | The Master API Key for LiteLLM.  If not specified, a random key in the `sk-...` format is generated.                                                                                   | N/A  |
-| `environmentSecrets`                                       | An optional array of Secret object names.  The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
-| `environmentConfigMaps`                                       | An optional array of ConfigMap object names.  The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables.  See below for an example Secret object.  | `[]`  |
-| `image.repository`                                         | LiteLLM Proxy image repository                                                                                                                                                        | `ghcr.io/berriai/litellm`  |
-| `image.pullPolicy`                                         | LiteLLM Proxy image pull policy                                                                                                                                                       | `IfNotPresent`  |
-| `image.tag`                                                | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published.                                                                             | `""`  |
-| `imagePullSecrets`                                         | Registry credentials for the LiteLLM and initContainer images.                                                                                                                        | `[]`  |
-| `serviceAccount.create`                                    | Whether or not to create a Kubernetes Service Account for this deployment.  The default is `false` because LiteLLM has no need to access the Kubernetes API.                          | `false`  |
-| `service.type`                                             | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                      | `ClusterIP`  |
-| `service.port`                                             | TCP port that the Kubernetes Service will listen on.  Also the TCP port within the Pod that the proxy will listen on.                                                                 | `4000`  |
-| `service.loadBalancerClass`                                | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                          | `""`  |
-| `ingress.*`                                                | See [values.yaml](./values.yaml) for example settings                                                                                                                                 | N/A  |
-| `proxyConfigMap.create`     | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it.                                                                                                                                      | `true` |
-| `proxyConfigMap.name`       | When `create=false`, name of the existing ConfigMap to mount.                                                                                                                                                  | `""`  |
-| `proxyConfigMap.key`        | Key in the ConfigMap that contains the proxy config file.                                                                                                                                                      | `"config.yaml"` |
-| `proxy_config.*`            | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` |
-| `extraContainers[]`         | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.
-| `pdb.enabled`                   | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment                                                                 | `false` |
-| `pdb.minAvailable`             | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
-| `pdb.maxUnavailable`           | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null`  |
-| `pdb.annotations`              | Extra metadata annotations to add to the PDB                                                                                   | `{}`    |
-| `pdb.labels`                   | Extra metadata labels to add to the PDB                                                                                        | `{}`    |
+| Name                        | Description                                                                                                                                                                                                                                   | Value                     |
+| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- |
+| `replicaCount`              | The number of LiteLLM Proxy pods to be deployed                                                                                                                                                                                               | `1`                       |
+| `masterkeySecretName`       | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name.                                                                                                              | N/A                       |
+| `masterkeySecretKey`        | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key.                                                                                                              | N/A                       |
+| `masterkey`                 | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated.                                                                                                                                           | N/A                       |
+| `environmentSecrets`        | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                                   | `[]`                      |
+| `environmentConfigMaps`     | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                             | `[]`                      |
+| `image.repository`          | LiteLLM Proxy image repository                                                                                                                                                                                                                | `docker.litellm.ai/berriai/litellm` |
+| `image.pullPolicy`          | LiteLLM Proxy image pull policy                                                                                                                                                                                                               | `IfNotPresent`            |
+| `image.tag`                 | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published.                                                                                                                                     | `""`                      |
+| `imagePullSecrets`          | Registry credentials for the LiteLLM and initContainer images.                                                                                                                                                                                | `[]`                      |
+| `serviceAccount.create`     | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API.                                                                                   | `false`                   |
+| `service.type`              | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.)                                                                                                                                                                              | `ClusterIP`               |
+| `service.port`              | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on.                                                                                                                          | `4000`                    |
+| `livenessProbe.*`           | Liveness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                  | See `values.yaml`         |
+| `readinessProbe.*`          | Readiness probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                 | See `values.yaml`         |
+| `startupProbe.*`            | Startup probe settings for the LiteLLM container (`path`, `periodSeconds`, `timeoutSeconds`, thresholds, and initial delay).                                                                                                                   | See `values.yaml`         |
+| `resources.*`               | CPU/memory requests and limits for the LiteLLM container.                                                                                                                                                                                       | `{}`                      |
+| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`)                                                                                                                                                  | `""`                      |
+| `ingress.labels`            | Additional labels for the Ingress resource                                                                                                                                                                                                    | `{}`                      |
+| `ingress.*`                 | See [values.yaml](./values.yaml) for example settings                                                                                                                                                                                         | N/A                       |
+| `proxyConfigMap.create`     | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it.                                                                                                                                                                     | `true`                    |
+| `proxyConfigMap.name`       | When `create=false`, name of the existing ConfigMap to mount.                                                                                                                                                                                 | `""`                      |
+| `proxyConfigMap.key`        | Key in the ConfigMap that contains the proxy config file.                                                                                                                                                                                     | `"config.yaml"`           |
+| `proxy_config.*`            | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A`                     |
+| `extraContainers[]`         | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy.                                                                                                                                                     |
+| `pdb.enabled`               | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment                                                                                                                                                                                 | `false`                   |
+| `pdb.minAvailable`          | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable)                                                                                                     | `null`                    |
+| `pdb.maxUnavailable`        | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable)                                                                                                    | `null`                    |
+| `pdb.annotations`           | Extra metadata annotations to add to the PDB                                                                                                                                                                                                  | `{}`                      |
+| `pdb.labels`                | Extra metadata labels to add to the PDB                                                                                                                                                                                                       | `{}`                      |
 
 #### Example `proxy_config` ConfigMap from values (default):
 
-
 ```
 proxyConfigMap:
   create: true
@@ -67,7 +73,6 @@ proxy_config:
 
 #### Example using existing `proxyConfigMap` instead of creating it:
 
-
 ```
 proxyConfigMap:
   create: false
@@ -77,8 +82,7 @@ proxyConfigMap:
 # proxy_config is ignored in this mode
 ```
 
-#### Example `environmentSecrets` Secret 
-
+#### Example `environmentSecrets` Secret
 
 ```
 apiVersion: v1
@@ -91,21 +95,23 @@ type: Opaque
 ```
 
 ### Database Settings
-| Name                                                       | Description                                                                                                                                                                           | Value |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| `db.useExisting`                                           | Use an existing Postgres database.  A Kubernetes Secret object must exist that contains credentials for connecting to the database.  An example secret object definition is provided below.  | `false`  |
-| `db.endpoint`                                              | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to.                                                                         | `localhost`  |
-| `db.database`                                              | If `db.useExisting` is `true`, the name of the existing database to connect to.                                                                                                       | `litellm`  |
-| `db.url`                                              | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value.                                                                                                       | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)`  |
-| `db.secret.name`                                           | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials.                                                                                           | `postgres`  |
-| `db.secret.usernameKey`                                    | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance.                                | `username`  |
-| `db.secret.passwordKey`                                    | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user.                                               | `password`  |
-| `db.useStackgresOperator`                                  | Not yet implemented.                                                                                                                                                                  | `false`  |
-| `db.deployStandalone`                                      | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart.  This is useful for getting started but doesn't provide HA or (by default) data backups.  | `true`  |
-| `postgresql.*`                                             | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart.   See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details.  See [values.yaml](./values.yaml) for the default configuration.  | See [values.yaml](./values.yaml) |
-| `postgresql.auth.*`                                        | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used.                                            | `NoTaGrEaTpAsSwOrD`  |
+
+| Name                      | Description                                                                                                                                                                                                                                                                               | Value                                                                                      |
+| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ |
+| `db.useExisting`          | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below.                                                                                                 | `false`                                                                                    |
+| `db.endpoint`             | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to.                                                                                                                                                                             | `localhost`                                                                                |
+| `db.database`             | If `db.useExisting` is `true`, the name of the existing database to connect to.                                                                                                                                                                                                           | `litellm`                                                                                  |
+| `db.url`                  | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value.                                                                                                                                                              | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` |
+| `db.secret.name`          | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials.                                                                                                                                                                                               | `postgres`                                                                                 |
+| `db.secret.usernameKey`   | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance.                                                                                                                                    | `username`                                                                                 |
+| `db.secret.passwordKey`   | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user.                                                                                                                                                   | `password`                                                                                 |
+| `db.useStackgresOperator` | Not yet implemented.                                                                                                                                                                                                                                                                      | `false`                                                                                    |
+| `db.deployStandalone`     | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups.                                                                                                  | `true`                                                                                     |
+| `postgresql.*`            | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml)                                                           |
+| `postgresql.auth.*`       | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used.                                                                                                                                                | `NoTaGrEaTpAsSwOrD`                                                                        |
 
 #### Example Postgres `db.useExisting` Secret
+
 ```yaml
 apiVersion: v1
 kind: Secret
@@ -143,7 +149,7 @@ metadata:
   name: litellm-env-secret
 type: Opaque
 data:
-  SOME_PASSWORD: cDZbUGVXeU5e0ZW    # base64 encoded
+  SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded
   ANOTHER_PASSWORD: AAZbUGVXeU5e0ZB # base64 encoded
 ```
 
@@ -153,23 +159,23 @@ Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472
 
 The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments.
 
-| Name                                                       | Description                                                                                                                                                                           | Value |
-| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- |
-| `migrationJob.enabled`                                     | Enable or disable the schema migration Job                                                                                                                                           | `true`  |
-| `migrationJob.backoffLimit`                                | Backoff limit for Job restarts                                                                                                                                                       | `4`  |
-| `migrationJob.ttlSecondsAfterFinished`                     | TTL for completed migration jobs                                                                                                                                                      | `120`  |
-| `migrationJob.annotations`                                 | Additional annotations for the migration job pod                                                                                                                                      | `{}`  |
-| `migrationJob.extraContainers`                             | Additional containers to run alongside the migration job                                                                                                                              | `[]`  |
-| `migrationJob.hooks.argocd.enabled`                       | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy)                                                                                  | `true`  |
-| `migrationJob.hooks.helm.enabled`                         | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy)                                                                 | `false`  |
-| `migrationJob.hooks.helm.weight`                          | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified.                                                                               | N/A  |
-
+| Name                                   | Description                                                                                                          | Value   |
+| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------- |
+| `migrationJob.enabled`                 | Enable or disable the schema migration Job                                                                           | `true`  |
+| `migrationJob.backoffLimit`            | Backoff limit for Job restarts                                                                                       | `4`     |
+| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs                                                                                     | `120`   |
+| `migrationJob.annotations`             | Additional annotations for the migration job pod                                                                     | `{}`    |
+| `migrationJob.extraContainers`         | Additional containers to run alongside the migration job                                                             | `[]`    |
+| `migrationJob.hooks.argocd.enabled`    | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy)                  | `true`  |
+| `migrationJob.hooks.helm.enabled`      | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` |
+| `migrationJob.hooks.helm.weight`       | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified.               | N/A     |
 
 ## Accessing the Admin UI
+
 When browsing to the URL published per the settings in `ingress.*`, you will
-be prompted for **Admin Configuration**.  The **Proxy Endpoint** is the internal
+be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal
 (from the `litellm` pod's perspective) URL published by the `<RELEASE>-litellm`
-Kubernetes Service.  If the deployment uses the default settings for this
+Kubernetes Service. If the deployment uses the default settings for this
 service, the **Proxy Endpoint** should be set to `http://<RELEASE>-litellm:4000`.
 
 The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey`
@@ -181,7 +187,8 @@ kubectl -n litellm get secret <RELEASE>-litellm-masterkey -o jsonpath="{.data.ma
 ```
 
 ## Admin UI Limitations
-At the time of writing, the Admin UI is unable to add models.  This is because
+
+At the time of writing, the Admin UI is unable to add models. This is because
 it would need to update the `config.yaml` file which is a exposed ConfigMap, and
-therefore, read-only.  This is a limitation of this helm chart, not the Admin UI
+therefore, read-only. This is a limitation of this helm chart, not the Admin UI
 itself.

deploy/charts/litellm-helm/templates/_helpers.tpl

@@ -61,6 +61,20 @@ Create the name of the service account to use
 {{- end }}
 {{- end }}
 
+{{/*
+Create the service account name used by migration jobs.
+When Helm hooks are enabled, pre-install/pre-upgrade hooks run before normal resources.
+If this chart is creating the ServiceAccount, it is not yet available for the hook job,
+so fall back to "default" (or an explicit override) to avoid a cyclic dependency.
+*/}}
+{{- define "litellm.migrationServiceAccountName" -}}
+{{- if and .Values.migrationJob.hooks.helm.enabled .Values.serviceAccount.create }}
+{{- default "default" .Values.migrationJob.serviceAccountName }}
+{{- else }}
+{{- include "litellm.serviceAccountName" . }}
+{{- end }}
+{{- end }}
+
 {{/*
 Get redis service name
 */}}

deploy/charts/litellm-helm/templates/configmap-litellm.yaml

@@ -6,4 +6,4 @@ metadata:
 data:
   config.yaml: |
 {{ .Values.proxy_config | toYaml | indent 6 }}
-{{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -6,13 +6,23 @@ metadata:
   name: {{ include "litellm.fullname" . }}
   labels:
     {{- include "litellm.labels" . | nindent 4 }}
+    {{- if .Values.deploymentLabels }}
+    {{- toYaml .Values.deploymentLabels | nindent 4 }}
+    {{- end }}
 spec:
-  {{- if not .Values.autoscaling.enabled }}
+  {{- if and (not .Values.keda.enabled) (not .Values.autoscaling.enabled) }}
   replicas: {{ .Values.replicaCount }}
   {{- end }}
+  {{- with .Values.strategy }}
+  strategy:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   selector:
     matchLabels:
       {{- include "litellm.selectorLabels" . | nindent 6 }}
+  {{- if .Values.deploymentMinReadySeconds }}
+  minReadySeconds: {{ .Values.deploymentMinReadySeconds }}
+  {{- end }}
   template:
     metadata:
       annotations:
@@ -35,6 +45,10 @@ spec:
       serviceAccountName: {{ include "litellm.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      {{- with .Values.extraInitContainers }}
+      initContainers:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       containers:
         - name: {{ include "litellm.name" . }}
           securityContext:
@@ -126,9 +140,20 @@ spec:
             - configMapRef:
                 name: {{ . }}
           {{- end }}
+          {{- if .Values.command }}
+          command: {{ toYaml .Values.command | nindent 12 }}
+          {{- end }}
+          {{- if .Values.args }}
+          args: {{ toYaml .Values.args | nindent 12 }}
+          {{- else }}
           args:
             - --config
             - /etc/litellm/config.yaml
+            {{ if .Values.numWorkers }}
+            - --num_workers
+            - {{ .Values.numWorkers | quote }}
+            {{- end }}
+          {{- end }}
           ports:
             - name: http
               containerPort: {{ .Values.service.port }}
@@ -140,23 +165,37 @@ spec:
             {{- end }}
           livenessProbe:
             httpGet:
-              path: /health/liveliness
+              path: {{ .Values.livenessProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+            initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.livenessProbe.successThreshold }}
+            failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
           readinessProbe:
             httpGet:
-              path: /health/readiness
+              path: {{ .Values.readinessProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+            initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
           startupProbe:
             httpGet:
-              path: /health/readiness
+              path: {{ .Values.startupProbe.path | quote }}
               port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
-            failureThreshold: 30
-            periodSeconds: 10
+            initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.startupProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.startupProbe.successThreshold }}
+            failureThreshold: {{ .Values.startupProbe.failureThreshold }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
             - name: litellm-config
-              mountPath: /etc/litellm/
+              mountPath: /etc/litellm/config.yaml
+              subPath: config.yaml
           {{ if .Values.securityContext.readOnlyRootFilesystem }}
             - name: tmp
               mountPath: /tmp
@@ -168,6 +207,10 @@ spec:
           {{- with .Values.volumeMounts }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.lifecycle }}
+          lifecycle:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
       {{- with .Values.extraContainers }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
@@ -208,3 +251,8 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 90 }}
+      {{- if .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml .Values.topologySpreadConstraints | nindent 8 }}
+      {{- end }}

deploy/charts/litellm-helm/templates/extra-resources.yaml

@@ -0,0 +1,6 @@
+{{- if .Values.extraResources }}
+{{- range .Values.extraResources }}
+---
+{{ toYaml . | nindent 0 }}
+{{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/ingress.yaml

@@ -18,6 +18,9 @@ metadata:
   name: {{ $fullName }}
   labels:
     {{- include "litellm.labels" . | nindent 4 }}
+    {{- with .Values.ingress.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.ingress.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}

deploy/charts/litellm-helm/templates/keda.yaml

@@ -0,0 +1,37 @@
+{{- if and .Values.keda.enabled (not .Values.autoscaling.enabled) }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: {{ include "litellm.fullname" . }}
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
+  {{- if .Values.keda.scaledObject.annotations }}
+  annotations: {{ toYaml .Values.keda.scaledObject.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  scaleTargetRef:
+    name: {{ include "litellm.fullname" . }}
+  pollingInterval: {{ .Values.keda.pollingInterval }}
+  cooldownPeriod: {{ .Values.keda.cooldownPeriod }}
+  minReplicaCount: {{ .Values.keda.minReplicas }}
+  maxReplicaCount: {{ .Values.keda.maxReplicas }}
+{{- with .Values.keda.fallback }}
+  fallback:
+    failureThreshold: {{ .failureThreshold | default 3 }}
+    replicas: {{ .replicas | default $.Values.keda.maxReplicas }}
+{{- end }}
+  triggers:
+{{- with .Values.keda.triggers }}
+  {{- toYaml . | nindent 2 }}
+{{- end }}
+  advanced:
+    restoreToOriginalReplicaCount: {{ .Values.keda.restoreToOriginalReplicaCount }}
+{{- if .Values.keda.behavior }}
+    horizontalPodAutoscalerConfig:
+      behavior:
+{{- with .Values.keda.behavior }}
+{{- toYaml . | nindent 8 }}
+{{- end }}
+
+{{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/migrations-job.yaml

@@ -22,6 +22,9 @@ spec:
     metadata:
       labels:
         {{- include "litellm.labels" . | nindent 8 }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
       annotations:
         {{- with .Values.migrationJob.annotations }}
         {{- toYaml . | nindent 8 }}
@@ -31,7 +34,11 @@ spec:
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+      serviceAccountName: {{ include "litellm.migrationServiceAccountName" . }}
+      {{- with .Values.migrationJob.extraInitContainers }}
+      initContainers:
+        {{- toYaml . | nindent 8 }}
+      {{- end }} 
       containers:
         - name: prisma-migrations
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default (printf "main-%s" .Chart.AppVersion) }}"

deploy/charts/litellm-helm/templates/servicemonitor.yaml

@@ -0,0 +1,39 @@
+{{- with .Values.serviceMonitor }}
+{{- if and (eq .enabled true) }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "litellm.fullname" $ }}
+  labels:
+    {{- include "litellm.labels" $ | nindent 4 }}
+    {{- if .labels }}
+    {{- toYaml .labels | nindent 4 }}
+    {{- end }}
+  {{- if .annotations }}
+  annotations:
+    {{- toYaml .annotations | nindent 4 }}
+  {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "litellm.selectorLabels" $ | nindent 6 }}
+  namespaceSelector:
+    matchNames:
+    # if not set, use the release namespace
+    {{- if not .namespaceSelector.matchNames }}
+    - {{ $.Release.Namespace | quote }}
+    {{- else }}
+    {{- toYaml .namespaceSelector.matchNames | nindent 4 }}
+    {{- end }}
+  endpoints:
+  - port: http
+    path: /metrics/
+    interval: {{ .interval }}
+    scrapeTimeout: {{ .scrapeTimeout }}
+    scheme: http
+  {{- if .relabelings }}
+    relabelings:
+{{- toYaml .relabelings | nindent 4 }}
+  {{- end }}
+{{- end }}
+{{- end }}

deploy/charts/litellm-helm/templates/tests/test-servicemonitor.yaml

@@ -0,0 +1,152 @@
+{{- if .Values.serviceMonitor.enabled }}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "litellm.fullname" . }}-test-servicemonitor"
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: test
+      image: bitnami/kubectl:latest
+      command: ['sh', '-c']
+      args:
+        - |
+          set -e
+          echo "🔍 Testing ServiceMonitor configuration..."
+          
+          # Check if ServiceMonitor exists
+          if ! kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} &>/dev/null; then
+            echo "❌ ServiceMonitor not found"
+            exit 1
+          fi
+          echo "✅ ServiceMonitor exists"
+          
+          # Get ServiceMonitor YAML
+          SM=$(kubectl get servicemonitor {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o yaml)
+          
+          # Test endpoint configuration
+          ENDPOINT_PORT=$(echo "$SM" | grep -A 5 "endpoints:" | grep "port:" | awk '{print $2}')
+          if [ "$ENDPOINT_PORT" != "http" ]; then
+            echo "❌ Endpoint port mismatch. Expected: http, Got: $ENDPOINT_PORT"
+            exit 1
+          fi
+          echo "✅ Endpoint port is correctly set to: $ENDPOINT_PORT"
+          
+          # Test endpoint path
+          ENDPOINT_PATH=$(echo "$SM" | grep -A 5 "endpoints:" | grep "path:" | awk '{print $2}')
+          if [ "$ENDPOINT_PATH" != "/metrics/" ]; then
+            echo "❌ Endpoint path mismatch. Expected: /metrics/, Got: $ENDPOINT_PATH"
+            exit 1
+          fi
+          echo "✅ Endpoint path is correctly set to: $ENDPOINT_PATH"
+          
+          # Test interval
+          INTERVAL=$(echo "$SM" | grep "interval:" | awk '{print $2}')
+          if [ "$INTERVAL" != "{{ .Values.serviceMonitor.interval }}" ]; then
+            echo "❌ Interval mismatch. Expected: {{ .Values.serviceMonitor.interval }}, Got: $INTERVAL"
+            exit 1
+          fi
+          echo "✅ Interval is correctly set to: $INTERVAL"
+          
+          # Test scrapeTimeout
+          TIMEOUT=$(echo "$SM" | grep "scrapeTimeout:" | awk '{print $2}')
+          if [ "$TIMEOUT" != "{{ .Values.serviceMonitor.scrapeTimeout }}" ]; then
+            echo "❌ ScrapeTimeout mismatch. Expected: {{ .Values.serviceMonitor.scrapeTimeout }}, Got: $TIMEOUT"
+            exit 1
+          fi
+          echo "✅ ScrapeTimeout is correctly set to: $TIMEOUT"
+          
+          # Test scheme
+          SCHEME=$(echo "$SM" | grep "scheme:" | awk '{print $2}')
+          if [ "$SCHEME" != "http" ]; then
+            echo "❌ Scheme mismatch. Expected: http, Got: $SCHEME"
+            exit 1
+          fi
+          echo "✅ Scheme is correctly set to: $SCHEME"
+          
+          {{- if .Values.serviceMonitor.labels }}
+          # Test custom labels
+          echo "🔍 Checking custom labels..."
+          {{- range $key, $value := .Values.serviceMonitor.labels }}
+          LABEL_VALUE=$(echo "$SM" | grep -A 20 "metadata:" | grep "{{ $key }}:" | awk '{print $2}')
+          if [ "$LABEL_VALUE" != "{{ $value }}" ]; then
+            echo "❌ Label {{ $key }} mismatch. Expected: {{ $value }}, Got: $LABEL_VALUE"
+            exit 1
+          fi
+          echo "✅ Label {{ $key }} is correctly set to: {{ $value }}"
+          {{- end }}
+          {{- end }}
+          
+          {{- if .Values.serviceMonitor.annotations }}
+          # Test annotations
+          echo "🔍 Checking annotations..."
+          {{- range $key, $value := .Values.serviceMonitor.annotations }}
+          ANNOTATION_VALUE=$(echo "$SM" | grep -A 10 "annotations:" | grep "{{ $key }}:" | awk '{print $2}')
+          if [ "$ANNOTATION_VALUE" != "{{ $value }}" ]; then
+            echo "❌ Annotation {{ $key }} mismatch. Expected: {{ $value }}, Got: $ANNOTATION_VALUE"
+            exit 1
+          fi
+          echo "✅ Annotation {{ $key }} is correctly set to: {{ $value }}"
+          {{- end }}
+          {{- end }}
+          
+          {{- if .Values.serviceMonitor.namespaceSelector.matchNames }}
+          # Test namespace selector
+          echo "🔍 Checking namespace selector..."
+          {{- range .Values.serviceMonitor.namespaceSelector.matchNames }}
+          if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ . }}"; then
+            echo "❌ Namespace {{ . }} not found in namespaceSelector"
+            exit 1
+          fi
+          echo "✅ Namespace {{ . }} found in namespaceSelector"
+          {{- end }}
+          {{- else }}
+          # Test default namespace selector (should be release namespace)
+          if ! echo "$SM" | grep -A 5 "namespaceSelector:" | grep -q "{{ .Release.Namespace }}"; then
+            echo "❌ Release namespace {{ .Release.Namespace }} not found in namespaceSelector"
+            exit 1
+          fi
+          echo "✅ Default namespace selector set to release namespace: {{ .Release.Namespace }}"
+          {{- end }}
+          
+          {{- if .Values.serviceMonitor.relabelings }}
+          # Test relabelings
+          echo "🔍 Checking relabelings configuration..."
+          if ! echo "$SM" | grep -q "relabelings:"; then
+            echo "❌ Relabelings section not found"
+            exit 1
+          fi
+          echo "✅ Relabelings section exists"
+          {{- range .Values.serviceMonitor.relabelings }}
+          {{- if .targetLabel }}
+          if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "targetLabel: {{ .targetLabel }}"; then
+            echo "❌ Relabeling targetLabel {{ .targetLabel }} not found"
+            exit 1
+          fi
+          echo "✅ Relabeling targetLabel {{ .targetLabel }} found"
+          {{- end }}
+          {{- if .action }}
+          if ! echo "$SM" | grep -A 50 "relabelings:" | grep -q "action: {{ .action }}"; then
+            echo "❌ Relabeling action {{ .action }} not found"
+            exit 1
+          fi
+          echo "✅ Relabeling action {{ .action }} found"
+          {{- end }}
+          {{- end }}
+          {{- end }}
+          
+          # Test selector labels match the service
+          echo "🔍 Checking selector labels match service..."
+          SVC_LABELS=$(kubectl get svc {{ include "litellm.fullname" . }} -n {{ .Release.Namespace }} -o jsonpath='{.metadata.labels}')
+          echo "Service labels: $SVC_LABELS"
+          echo "✅ Selector labels validation passed"
+          
+          echo ""
+          echo "🎉 All ServiceMonitor tests passed successfully!"
+  serviceAccountName: {{ include "litellm.serviceAccountName" . }}
+  restartPolicy: Never
+{{- end }}
+

deploy/charts/litellm-helm/tests/deployment_command_args_labels_tests.yaml

@@ -0,0 +1,68 @@
+suite: test deployment command, args, and deploymentLabels
+templates:
+  - deployment.yaml
+  - configmap-litellm.yaml
+tests:
+  - it: should override args when custom args specified
+    template: deployment.yaml
+    set:
+      args:
+        - --custom-arg1
+        - value1
+        - --custom-arg2
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].args
+          value:
+            - --custom-arg1
+            - value1
+            - --custom-arg2
+  - it: should set custom command when specified
+    template: deployment.yaml
+    set:
+      command:
+        - /bin/sh
+        - -c
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].command
+          value:
+            - /bin/sh
+            - -c
+  - it: should set custom command and args together
+    template: deployment.yaml
+    set:
+      command:
+        - python
+        - -u
+      args:
+        - my_script.py
+        - --verbose
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].command
+          value:
+            - python
+            - -u
+      - equal:
+          path: spec.template.spec.containers[0].args
+          value:
+            - my_script.py
+            - --verbose
+  - it: should add deploymentLabels to deployment metadata
+    template: deployment.yaml
+    set:
+      deploymentLabels:
+        environment: production
+        team: platform
+        version: v1.2.3
+    asserts:
+      - equal:
+          path: metadata.labels.environment
+          value: production
+      - equal:
+          path: metadata.labels.team
+          value: platform
+      - equal:
+          path: metadata.labels.version
+          value: v1.2.3

deploy/charts/litellm-helm/tests/deployment_tests.yaml

@@ -136,4 +136,186 @@ tests:
           path: spec.template.spec.containers[0].volumeMounts
           content:
             name: litellm-config
-            mountPath: /etc/litellm/
+            mountPath: /etc/litellm/config.yaml
+            subPath: config.yaml
+  - it: should work with lifecycle hooks
+    template: deployment.yaml
+    set:
+      lifecycle:
+        preStop:
+          exec:
+            command:
+              - /bin/sh
+              - -c
+              - echo "Container stopping"
+    asserts:
+      - exists:
+          path: spec.template.spec.containers[0].lifecycle
+      - equal:
+          path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[0]
+          value: /bin/sh
+      - equal:
+          path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[1]
+          value: -c
+      - equal:
+          path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
+          value: echo "Container stopping"
+  - it: should render background health check settings from proxy_config.general_settings
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.general_settings.background_health_checks: true
+      proxy_config.general_settings.health_check_interval: 240
+      proxy_config.general_settings.health_check_concurrency: 16
+      proxy_config.general_settings.health_check_details: false
+    asserts:
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*background_health_checks:\s*true$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_interval:\s*240$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_concurrency:\s*16$'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: '(?m)^\s*health_check_details:\s*false$'
+  - it: should allow overriding liveness, readiness, and startup probes
+    template: deployment.yaml
+    set:
+      livenessProbe:
+        path: /custom/livez
+        initialDelaySeconds: 5
+        periodSeconds: 15
+        timeoutSeconds: 5
+        successThreshold: 1
+        failureThreshold: 5
+      readinessProbe:
+        path: /custom/readyz
+        initialDelaySeconds: 10
+        periodSeconds: 20
+        timeoutSeconds: 6
+        successThreshold: 1
+        failureThreshold: 6
+      startupProbe:
+        path: /custom/startupz
+        initialDelaySeconds: 15
+        periodSeconds: 25
+        timeoutSeconds: 7
+        successThreshold: 1
+        failureThreshold: 40
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /custom/livez
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 5
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /custom/readyz
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 6
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.httpGet.path
+          value: /custom/startupz
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.failureThreshold
+          value: 40
+  - it: should render container resources from values
+    template: deployment.yaml
+    set:
+      resources:
+        limits:
+          cpu: 500m
+          memory: 2Gi
+        requests:
+          cpu: 250m
+          memory: 1Gi
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.cpu
+          value: 500m
+      - equal:
+          path: spec.template.spec.containers[0].resources.limits.memory
+          value: 2Gi
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.cpu
+          value: 250m
+      - equal:
+          path: spec.template.spec.containers[0].resources.requests.memory
+          value: 1Gi
+  - it: should keep default probes and empty resources unchanged
+    template: deployment.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.httpGet.path
+          value: /health/liveliness
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].livenessProbe.failureThreshold
+          value: 3
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.httpGet.path
+          value: /health/readiness
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].readinessProbe.failureThreshold
+          value: 3
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.httpGet.path
+          value: /health/readiness
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds
+          value: 0
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.periodSeconds
+          value: 10
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.timeoutSeconds
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.successThreshold
+          value: 1
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe.failureThreshold
+          value: 30
+      - equal:
+          path: spec.template.spec.containers[0].resources
+          value: {}
+  - it: should be able to set minReadySeconds
+    template: deployment.yaml
+    set:
+      deploymentMinReadySeconds: 5
+    asserts:
+      - equal:
+          path: spec.minReadySeconds
+          value: 5
+  - it: should have minReadySeconds absent when deploymentMinReadySeconds is not set
+    template: deployment.yaml
+    asserts:
+      - notExists:
+          path: spec.minReadySeconds