scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

Merge branch 'main' into litellm_live_api_passthrough

Browse source
This commit is contained in:
Sameer Kankute 2025-09-28 09:45:57 +05:30 • committed by GitHub
commit f510ac15f8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
226 changed files with 3995 additions and 2063 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
.circleci/config.yml
View file

@ -51,9 +51,36 @@ jobs:
command: |
python -m pytest tests/windows_tests/test_litellm_on_windows.py -v
mypy_linting:
docker:
- image: cimg/python:3.12
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: medium
steps:
- checkout
- setup_google_dns
- run:
name: Install Dependencies
command: |
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
pip uninstall fastuuid -y
pip install "mypy==1.18.2"
- run:
name: MyPy Type Checking
command: |
cd litellm
# Use the same approach as GitHub Actions, explicitly exclude fastuuid to avoid segfaults
python -m mypy .
cd ..
no_output_timeout: 10m
local_testing:
docker:
- image: cimg/python:3.11
- image: cimg/python:3.12
auth:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
@ -79,7 +106,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install "mypy==1.15.0"
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
@ -140,20 +167,6 @@ jobs:
python -m pip install black
python -m black .
cd ..
- run:
name: Linting Testing
command: |
cd litellm
pip install "cryptography>=43.0.1"
python -m pip install types-requests types-setuptools types-redis types-PyYAML
if ! python -m mypy . \
--config-file mypy.ini \
--ignore-missing-imports \
--no-incremental; then
echo "mypy detected errors"
exit 1
fi
cd ..
# Run pytest and generate JUnit XML report
- run:
@ -161,7 +174,7 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml -x --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4
python -m pytest -vv tests/local_testing --cov=litellm --cov-report=xml --junitxml=test-results/junit.xml --durations=5 -k "not test_python_38.py and not test_basic_python_version.py and not router and not assistants and not langfuse and not caching and not cache" -n 4
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -205,7 +218,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install mypy
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
@ -312,7 +325,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install mypy
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
@ -471,7 +484,7 @@ jobs:
command: |
pwd
ls
python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -x -v --junitxml=test-results/junit.xml --durations=5
python -m pytest tests/local_testing --cov=litellm --cov-report=xml -vv -k "router" -v --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -553,14 +566,14 @@ jobs:
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
- run:
name: Install Python 3.9
name: Install Python 3.13
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda create -n myenv python=3.13 -y
conda activate myenv
python --version
- run:
@ -575,7 +588,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install mypy
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
@ -669,7 +682,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install "pytest-cov==5.0.0"
pip install mypy
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install "google-genai==1.22.0"
@ -817,7 +830,7 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -x -v --junitxml=test-results/junit.xml --durations=5 -n 4
python -m pytest -vv tests/llm_translation --cov=litellm --cov-report=xml -v --junitxml=test-results/junit.xml --durations=5 -n 4
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1049,7 +1062,7 @@ jobs:
command: |
pwd
ls
python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -x -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8
python -m pytest -vv tests/test_litellm --cov=litellm --cov-report=xml -s -v --junitxml=test-results/junit-litellm.xml --durations=10 -n 8
no_output_timeout: 120m
- run:
name: Rename the coverage files
@ -1187,6 +1200,7 @@ jobs:
pip install "pytest-cov==5.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pytest-mock
# Run pytest and generate JUnit XML report
- run:
name: Run tests
@ -1637,7 +1651,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install mypy
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
@ -1775,7 +1789,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install mypy
pip install "mypy==1.18.2"
pip install "jsonlines==4.0.0"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
@ -1917,7 +1931,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install mypy
pip install "mypy==1.18.2"
pip install "google-generativeai==0.3.2"
pip install "google-cloud-aiplatform==1.43.0"
pip install pyarrow
@ -2420,7 +2434,7 @@ jobs:
pip install "pytest-retry==1.6.3"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install mypy
pip install "mypy==1.18.2"
- run:
name: Build Docker image
command: |
@ -2525,7 +2539,7 @@ jobs:
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install "boto3==1.36.0"
pip install mypy
pip install "mypy==1.18.2"
pip install pyarrow
pip install numpydoc
pip install prisma
@ -2914,7 +2928,7 @@ jobs:
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install mypy
pip install "mypy==1.18.2"
pip install pyarrow
pip install numpydoc
pip install prisma
@ -3078,6 +3092,12 @@ workflows:
only:
- main
- /litellm_.*/
- mypy_linting:
filters:
branches:
only:
- main
- /litellm_.*/
- local_testing:
filters:
branches:
@ -3324,6 +3344,7 @@ workflows:
- main
- publish_to_pypi:
requires:
- mypy_linting
- local_testing
- build_and_test
- e2e_openai_endpoints

11
.devcontainer/devcontainer.json
View file

@ -11,7 +11,12 @@
// },
// Features to add to the dev container. More info: https://containers.dev/features.
// "features": {},
"features": {
"ghcr.io/devcontainers/features/node:1": {
"version": "lts"
},
"ghcr.io/devcontainers/features/docker-in-docker:2": {}
},
// Configure tool-specific properties.
"customizations": {
@ -30,7 +35,7 @@
// Use 'forwardPorts' to make a list of ports inside the container available locally.
"forwardPorts": [4000],
"containerEnv": {
"LITELLM_LOG": "DEBUG"
},
@ -48,5 +53,5 @@
// "remoteUser": "litellm",
// Use 'postCreateCommand' to run commands after the container is created.
"postCreateCommand": "pipx install poetry && poetry install -E extra_proxy -E proxy"
"postCreateCommand": "bash ./.devcontainer/post-create.sh"
}

17
.devcontainer/post-create.sh Normal file
View file

@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -e
echo "[post-create] Installing poetry via pip"
python -m pip install --upgrade pip
python -m pip install poetry
echo "[post-create] Installing Python dependencies (poetry)"
poetry install --with dev --extras proxy
echo "[post-create] Generating Prisma client"
poetry run prisma generate
echo "[post-create] Installing npm dependencies"
cd ui/litellm-dashboard && npm install --no-audit --no-fund
echo "[post-create] Done"

19
.github/workflows/test-linting.yml vendored
View file

@ -11,6 +11,9 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
clean: true
- name: Set up Python
uses: actions/setup-python@v4
@ -20,6 +23,11 @@ jobs:
- name: Install Poetry
uses: snok/install-poetry@v1
- name: Clean Python cache
run: |
find . -type d -name "__pycache__" -exec rm -rf {} + || true
find . -name "*.pyc" -delete || true
- name: Install dependencies
run: |
poetry install --with dev
@ -31,6 +39,15 @@ jobs:
poetry run black .
cd ..
- name: Debug - Check file state
run: |
echo "Current branch:"
git branch --show-current
echo "Last 3 commits:"
git log --oneline -3
echo "File content around line 43:"
head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
- name: Run Ruff linting
run: |
cd litellm
@ -44,7 +61,7 @@ jobs:
- name: Run MyPy type checking
run: |
cd litellm
poetry run mypy . --ignore-missing-imports
poetry run mypy .
cd ..
- name: Check for circular imports

2
.github/workflows/test-litellm.yml vendored
View file

@ -40,4 +40,4 @@ jobs:
cd ..
- name: Run tests
run: |
poetry run pytest tests/test_litellm -x -vv -n 4
poetry run pytest tests/test_litellm --tb=short -vv --maxfail=10 -n 4

5
Dockerfile
View file

@ -15,7 +15,7 @@ USER root
RUN apk add --no-cache gcc python3-dev openssl openssl-dev
RUN pip install --upgrade pip && \
RUN pip install --upgrade pip>=24.3.1 && \
pip install build
# Copy the current directory contents into the container at /app
@ -50,6 +50,9 @@ USER root
# Install runtime dependencies
RUN apk add --no-cache openssl tzdata
# Upgrade pip to fix CVE-2025-8869
RUN pip install --upgrade pip>=24.3.1
WORKDIR /app
# Copy the current directory contents into the container at /app
COPY . .

32
ci_cd/security_scans.sh
View file

@ -50,12 +50,12 @@ run_grype_scans() {
# Build and scan Dockerfile.database
echo "Building and scanning Dockerfile.database..."
docker build -t litellm-database:latest -f ./docker/Dockerfile.database .
docker build --no-cache -t litellm-database:latest -f ./docker/Dockerfile.database .
grype litellm-database:latest --fail-on critical
# Build and scan main Dockerfile
echo "Building and scanning main Dockerfile..."
docker build -t litellm:latest .
docker build --no-cache -t litellm:latest .
grype litellm:latest --fail-on critical
# Restore original .dockerignore
@ -66,18 +66,34 @@ run_grype_scans() {
echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
echo "Using locally built image: litellm:latest"
# Run grype scan and check for vulnerabilities with CVSS >= 4.0
# Allowlist of CVEs to be ignored in failure threshold/reporting
# - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
ALLOWED_CVES=(
"CVE-2025-8869"
)
# Build JSON array of allowlisted CVE IDs for jq
ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
echo "Checking for vulnerabilities with CVSS score >= 4.0..."
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq -r '.matches[] | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) | .vulnerability.id' | wc -l)
echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| .vulnerability.id' | wc -l)
if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
echo "ERROR: Found $HIGH_SEVERITY_COUNT vulnerabilities with CVSS score >= 4.0 in litellm:latest"
echo "Detailed vulnerability report:"
grype litellm:latest -o json | jq -r '
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
(.matches[] | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) |
[.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description]) |
@tsv' | column -t -s $'\t'
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
| @tsv' | column -t -s $'\t'
exit 1
else
echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"

110
cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md
View file

@ -26,7 +26,7 @@ git diff <previous_commit_hash> HEAD -- model_prices_and_context_window.json
### 2. Release Notes Structure
Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable):
Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable, v1.77.5-stable):
```markdown
---
@ -41,7 +41,7 @@ hide_table_of_contents: false
[Docker and pip installation tabs]
## Key Highlights
[3-5 bullet points of major features]
[3-5 bullet points of major features - prioritize MCP OAuth 2.0, scheduled key rotations, and major model updates]
## New Models / Updated Models
#### New Model Support
@ -65,26 +65,32 @@ hide_table_of_contents: false
## Management Endpoints / UI
#### Features
[UI and management features]
[UI and management features - group by functionality like Proxy CLI Auth, Virtual Keys, Models + Endpoints]
#### Bugs
[Management-related bug fixes]
## Logging / Guardrail Integrations
## Logging / Guardrail / Prompt Management Integrations
#### Features
[Organized by integration provider with proper doc links]
#### Guardrails
[Guardrail-specific features and fixes]
#### New Integration
[Major new integrations]
#### Prompt Management
[Prompt management integrations like BitBucket]
## Spend Tracking, Budgets and Rate Limiting
[Cost tracking, service tier pricing, rate limiting improvements]
## MCP Gateway
[MCP-specific features, OAuth 2.0, configuration improvements]
## Performance / Loadbalancing / Reliability improvements
[Infrastructure improvements]
[Infrastructure improvements, memory fixes, performance optimizations]
## General Proxy Improvements
[Other proxy-related changes]
## Documentation Updates
[Documentation improvements, guides, corrections - separate section for visibility]
## New Contributors
[List of first-time contributors]
@ -101,6 +107,11 @@ hide_table_of_contents: false
- CPU usage optimizations
- Timeout controls
- Worker configuration
- Memory leak fixes
- Cache performance improvements
- Database connection management
- Dependency management (fastuuid, etc.)
- Configuration management
**New Models/Updated Models:**
- Extract from model_prices_and_context_window.json diff
@ -132,20 +143,32 @@ hide_table_of_contents: false
- Dashboard improvements
- Team management
- Key management
- Proxy CLI authentication and improvements
- Virtual key management and scheduled rotations
- SSO configuration fixes
- Admin settings updates
- Management routes and endpoints
**Logging / Guardrail Integrations:**
**Logging / Guardrail / Prompt Management Integrations:**
- **Structure:**
- `#### Features` - organized by integration provider with proper doc links
- `#### Guardrails` - guardrail-specific features and fixes
- `#### Prompt Management` - prompt management integrations
- `#### New Integration` - major new integrations
- **Integration Categories:**
- **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes
- **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features
- **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements
- **[PostHog](../../docs/observability/posthog)** - observability integration
- **[SQS](../../docs/proxy/logging#sqs)** - SQS logging features
- **[Opik](../../docs/proxy/logging#opik)** - Opik integration improvements
- Other logging providers with proper doc links
- **Guardrail Categories:**
- LakeraAI, Presidio, Noma, and other guardrail providers
- **Prompt Management:**
- BitBucket, GitHub, and other prompt management integrations
- Use bullet points under each provider for multiple features
- Separate logging features from guardrails clearly
- Separate logging features from guardrails and prompt management clearly
### 4. Documentation Linking Strategy
@ -189,15 +212,26 @@ From git diff analysis, create tables like:
- `[Perf]`, `Performance`, `RPS` → Performance Improvements
- `[Bug]`, `[Bug Fix]`, `Fix` → Bug Fixes section
- `[Feat]`, `[Feature]`, `Add support` → Features section
- `[Docs]` → Documentation (usually exclude from main sections)
- `[Docs]` → Documentation Updates section
- Provider names (Gemini, OpenAI, etc.) → Group under provider
- `MCP`, `oauth`, `Model Context Protocol` → MCP Gateway
- `service_tier`, `priority`, `cost tracking` → Spend Tracking, Budgets and Rate Limiting
**By PR Content Analysis:**
- New model additions → New Models section
- UI changes → Management Endpoints/UI
- Logging/observability → Logging/Guardrail Integrations
- Rate limiting/budgets → Performance/Reliability
- Authentication → Management Endpoints
- Logging/observability → Logging/Guardrail/Prompt Management Integrations
- Rate limiting/budgets → Spend Tracking, Budgets and Rate Limiting
- Authentication → Management Endpoints/UI
- MCP-related changes → MCP Gateway
- Documentation updates → Documentation Updates
- Performance/memory fixes → Performance/Loadbalancing/Reliability improvements
**Special Categorization Rules:**
- **Service tier pricing** (OpenAI priority/flex) → Spend Tracking section (NOT provider features)
- **Cost breakdown in logging** → Spend Tracking section
- **MCP configuration/OAuth** → MCP Gateway (NOT General Proxy Improvements)
- **All documentation PRs** → Documentation Updates section for visibility
### 7. Writing Style Guidelines
@ -226,6 +260,18 @@ From git diff analysis, create tables like:
- Ensure model pricing is accurate
- Confirm provider names are consistent
- Review for typos and formatting issues
- **Count PRs by section** - Provide final count like:
```
## MM/DD/YYYY
* New Models / Updated Models: XX
* LLM API Endpoints: XX
* Management Endpoints / UI: XX
* Logging / Guardrail / Prompt Management Integrations: XX
* Spend Tracking, Budgets and Rate Limiting: XX
* MCP Gateway: XX
* Performance / Loadbalancing / Reliability improvements: XX
* Documentation Updates: XX
```
### 9. Common Patterns to Follow
@ -295,6 +341,40 @@ This release has a known issue...
- Complex configuration options
- Migration requirements
### 11. New Sections and Categories (Added in v1.77.5)
**MCP Gateway Section:**
- All MCP-related changes go here (not in General Proxy Improvements)
- OAuth 2.0 flow improvements
- MCP configuration and tools
- Server management features
**Spend Tracking, Budgets and Rate Limiting Section:**
- Service tier pricing (OpenAI priority/flex pricing)
- Cost tracking and breakdown features
- Rate limiting improvements (Parallel Request Limiter v3)
- Priority reservation fixes
- Metadata handling for rate limiting
**Documentation Updates Section:**
- Create separate section for all documentation improvements
- Include provider documentation fixes
- Model reference updates
- New guides and tutorials
- Documentation corrections and clarifications
- This gives documentation changes proper visibility
**Management Endpoints / UI Grouping:**
- Group related features under sub-categories:
- **Proxy CLI Auth** - CLI authentication improvements
- **Virtual Keys** - Key rotation and management
- **Models + Endpoints** - Provider and endpoint management
**Logging Section Expansion:**
- Rename to "Logging / Guardrail / Prompt Management Integrations"
- Add **Prompt Management** subsection for BitBucket, GitHub integrations
- Keep guardrails separate from logging features
## Example Command Workflow
```bash

89
docs/my-website/docs/pass_through/azure_passthrough.md Normal file
View file

@ -0,0 +1,89 @@
# Azure Passthrough
Pass-through endpoints for `/azure`
## Overview
| Feature | Supported | Notes |
|-------|-------|-------|
| Cost Tracking | ❌ | Not supported |
| Logging | ✅ | Works across all integrations |
| Streaming | ✅ | Fully supported |
### When to use this?
- For most use cases, you should use the [native LiteLLM Azure OpenAI Integration](../providers/azure/azure) (`/chat/completions`, `/embeddings`, `/completions`, `/images`, etc.)
- Use this passthrough to call newer or less common Azure OpenAI endpoints that LiteLLM doesn't fully support yet, such as `/assistants`, `/threads`, `/vector_stores`
Simply replace your Azure endpoint (e.g. `https://<your-resource-name>.openai.azure.com`) with `LITELLM_PROXY_BASE_URL/azure`
## Usage Examples
### Assistants API
#### Create Azure OpenAI Client
Make sure you do the following:
- Point `azure_endpoint` to your `LITELLM_PROXY_BASE_URL/azure`
- Use your `LITELLM_API_KEY` as the `api_key`
```python
import openai
client = openai.AzureOpenAI(
azure_endpoint="http://0.0.0.0:4000/azure", # <your-proxy-url>/azure
api_key="sk-anything", # <your-proxy-api-key>
api_version="2024-05-01-preview" # required Azure API version
)
```
#### Create an Assistant
```python
assistant = client.beta.assistants.create(
name="Math Tutor",
instructions="You are a math tutor. Help solve equations.",
model="gpt-4o",
)
```
#### Create a Thread
```python
thread = client.beta.threads.create()
```
#### Add a Message to the Thread
```python
message = client.beta.threads.messages.create(
thread_id=thread.id,
role="user",
content="Solve 3x + 11 = 14",
)
```
#### Run the Assistant
```python
run = client.beta.threads.runs.create(
thread_id=thread.id,
assistant_id=assistant.id,
)
# Check run status
run_status = client.beta.threads.runs.retrieve(
thread_id=thread.id,
run_id=run.id
)
```
#### Retrieve Messages
```python
messages = client.beta.threads.messages.list(
thread_id=thread.id
)
```
#### Delete the Assistant
```python
client.beta.assistants.delete(assistant.id)
```

BIN
docs/my-website/img/release_notes/perf_imp.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.2 MiB

BIN
docs/my-website/img/release_notes/quota.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 276 KiB

22
docs/my-website/release_notes/v1.77.3-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[Preview] v1.77.3-stable - Priority Based Rate Limiting"
title: "v1.77.3-stable - Priority Based Rate Limiting"
slug: "v1-77-3"
date: 2025-09-21T10:00:00
authors:
@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.77.3.rc.1
ghcr.io/berriai/litellm:v1.77.3-stable
```
</TabItem>
@ -51,11 +51,27 @@ pip install litellm==1.77.3
## Priority Quota Reservation
This release adds support for priority quota reservation. This allows Proxy Admins to reserve specific percentages of model capacity for different use cases.
This is great for use cases where you want to ensure your realtime use cases must always get priority responses and background development jobs can take longer.
<Image img={require('../../img/release_notes/quota.png')} style={{ width: '800px', height: 'auto' }} />
<br/>
This release adds support for priority quota reservation. This allows **Proxy Admins** to reserve TPM/RPM capacity for keys based on metadata priority levels, ensuring critical production workloads get guaranteed access regardless of development traffic volume.
Get started [here](../../docs/proxy/dynamic_rate_limit#priority-quota-reservation)
<iframe width="700" height="500" src="https://www.loom.com/embed/1b54b93139ee415d959402cc0629f3f7" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## +550 RPS Performance Improvements
<Image img={require('../../img/release_notes/perf_imp.png')} style={{ width: '800px', height: 'auto' }} />
<br/>
This release delivers significant RPS improvements through targeted optimizations.
We've achieved a +500 RPS boost by fixing cache type inconsistencies that were causing frequent cache misses, plus an additional +50 RPS by removing unnecessary coroutine checks from the hot path.
## New Models / Updated Models

285
docs/my-website/release_notes/v1.77.5-stable/index.md Normal file
View file

@ -0,0 +1,285 @@
---
title: "[Preview] v1.77.5-stable - MCP OAuth 2.0 Support"
slug: "v1-77-5"
date: 2025-09-29T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
```
</TabItem>
</Tabs>
---
## Key Highlights
- **MCP OAuth 2.0 Support** - Enhanced authentication for Model Context Protocol integrations
- **Scheduled Key Rotations** - Automated key rotation capabilities for enhanced security
- **New Gemini 2.5 Flash & Flash-lite Models** - Latest September 2025 preview models with improved pricing and features
- **Performance Improvements** - Critical InMemoryCache unbounded growth resolution
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| Gemini | `gemini-2.5-flash-preview-09-2025` | 1M | $0.30 | $2.50 | Chat, reasoning, vision, audio |
| Gemini | `gemini-2.5-flash-lite-preview-09-2025` | 1M | $0.10 | $0.40 | Chat, reasoning, vision, audio |
| Gemini | `gemini-flash-latest` | 1M | $0.30 | $2.50 | Chat, reasoning, vision, audio |
| Gemini | `gemini-flash-lite-latest` | 1M | $0.10 | $0.40 | Chat, reasoning, vision, audio |
| DeepSeek | `deepseek-chat` | 131K | $0.60 | $1.70 | Chat, function calling, caching |
| DeepSeek | `deepseek-reasoner` | 131K | $0.60 | $1.70 | Chat, reasoning |
| Bedrock | `deepseek.v3-v1:0` | 164K | $0.58 | $1.68 | Chat, reasoning, function calling |
| Azure | `azure/gpt-5-codex` | 272K | $1.25 | $10.00 | Responses API, reasoning, vision |
| OpenAI | `gpt-5-codex` | 272K | $1.25 | $10.00 | Responses API, reasoning, vision |
| SambaNova | `sambanova/DeepSeek-V3.1` | 33K | $3.00 | $4.50 | Chat, reasoning, function calling |
| SambaNova | `sambanova/gpt-oss-120b` | 131K | $3.00 | $4.50 | Chat, reasoning, function calling |
| Bedrock | `qwen.qwen3-coder-480b-a35b-v1:0` | 262K | $0.22 | $1.80 | Chat, reasoning, function calling |
| Bedrock | `qwen.qwen3-235b-a22b-2507-v1:0` | 262K | $0.22 | $0.88 | Chat, reasoning, function calling |
| Bedrock | `qwen.qwen3-coder-30b-a3b-v1:0` | 262K | $0.15 | $0.60 | Chat, reasoning, function calling |
| Bedrock | `qwen.qwen3-32b-v1:0` | 131K | $0.15 | $0.60 | Chat, reasoning, function calling |
| Vertex AI | `vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas` | 262K | $0.15 | $1.20 | Chat, function calling |
| Vertex AI | `vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas` | 262K | $0.15 | $1.20 | Chat, function calling |
| Vertex AI | `vertex_ai/deepseek-ai/deepseek-v3.1-maas` | 164K | $1.35 | $5.40 | Chat, reasoning, function calling |
| OpenRouter | `openrouter/x-ai/grok-4-fast:free` | 2M | $0.00 | $0.00 | Chat, reasoning, function calling |
| XAI | `xai/grok-4-fast-reasoning` | 2M | $0.20 | $0.50 | Chat, reasoning, function calling |
| XAI | `xai/grok-4-fast-non-reasoning` | 2M | $0.20 | $0.50 | Chat, function calling |
#### Features
- **[Gemini](../../docs/providers/gemini)**
- Added Gemini 2.5 Flash and Flash-lite preview models (September 2025 release) with improved pricing - [PR #14948](https://github.com/BerriAI/litellm/pull/14948)
- Added new Anthropic web fetch tool support - [PR #14951](https://github.com/BerriAI/litellm/pull/14951)
- **[XAI](../../docs/providers/xai)**
- Add xai/grok-4-fast models - [PR #14833](https://github.com/BerriAI/litellm/pull/14833)
- **[Anthropic](../../docs/providers/anthropic)**
- Updated Claude Sonnet 4 configs to reflect million-token context window pricing - [PR #14639](https://github.com/BerriAI/litellm/pull/14639)
- Added supported text field to anthropic citation response - [PR #14164](https://github.com/BerriAI/litellm/pull/14164)
- **[Bedrock](../../docs/providers/bedrock)**
- Added support for Qwen models family & Deepseek 3.1 to Amazon Bedrock - [PR #14845](https://github.com/BerriAI/litellm/pull/14845)
- Support requestMetadata in Bedrock Converse API - [PR #14570](https://github.com/BerriAI/litellm/pull/14570)
- **[Vertex AI](../../docs/providers/vertex)**
- Added vertex_ai/qwen models and azure/gpt-5-codex - [PR #14844](https://github.com/BerriAI/litellm/pull/14844)
- Update vertex ai qwen model pricing - [PR #14828](https://github.com/BerriAI/litellm/pull/14828)
- Vertex AI Context Caching: use Vertex ai API v1 instead of v1beta1 and accept 'cachedContent' param - [PR #14831](https://github.com/BerriAI/litellm/pull/14831)
- **[SambaNova](../../docs/providers/sambanova)**
- Add sambanova deepseek v3.1 and gpt-oss-120b - [PR #14866](https://github.com/BerriAI/litellm/pull/14866)
- **[OpenAI](../../docs/providers/openai)**
- Fix inconsistent token configs for gpt-5 models - [PR #14942](https://github.com/BerriAI/litellm/pull/14942)
- GPT-3.5-Turbo price updated - [PR #14858](https://github.com/BerriAI/litellm/pull/14858)
- **[OpenRouter](../../docs/providers/openrouter)**
- Add gpt-5 and gpt-5-codex to OpenRouter cost map - [PR #14879](https://github.com/BerriAI/litellm/pull/14879)
- **[VLLM](../../docs/providers/vllm)**
- Fix vllm passthrough - [PR #14778](https://github.com/BerriAI/litellm/pull/14778)
- **[Flux](../../docs/image_generation)**
- Support flux image edit - [PR #14790](https://github.com/BerriAI/litellm/pull/14790)
### Bug Fixes
- **[Anthropic](../../docs/providers/anthropic)**
- Fix: Support claude code auth via subscription (anthropic) - [PR #14821](https://github.com/BerriAI/litellm/pull/14821)
- Fix Anthropic streaming IDs - [PR #14965](https://github.com/BerriAI/litellm/pull/14965)
- Revert incorrect changes to sonnet-4 max output tokens - [PR #14933](https://github.com/BerriAI/litellm/pull/14933)
- **[OpenAI](../../docs/providers/openai)**
- Fix a bug where openai image edit silently ignores multiple images - [PR #14893](https://github.com/BerriAI/litellm/pull/14893)
- **[VLLM](../../docs/providers/vllm)**
- Fix: vLLM provider's rerank endpoint from /v1/rerank to /rerank - [PR #14938](https://github.com/BerriAI/litellm/pull/14938)
#### New Provider Support
- **[W&B Inference](../../docs/providers/wandb)**
- Add W&B Inference to LiteLLM - [PR #14416](https://github.com/BerriAI/litellm/pull/14416)
---
## LLM API Endpoints
#### Features
- **General**
- Add SDK support for additional headers - [PR #14761](https://github.com/BerriAI/litellm/pull/14761)
- Add shared_session parameter for aiohttp ClientSession reuse - [PR #14721](https://github.com/BerriAI/litellm/pull/14721)
#### Bugs
- **General**
- Fix: Streaming tool call index assignment for multiple tool calls - [PR #14587](https://github.com/BerriAI/litellm/pull/14587)
- Fix load credentials in token counter proxy - [PR #14808](https://github.com/BerriAI/litellm/pull/14808)
---
## Management Endpoints / UI
#### Features
- **Proxy CLI Auth**
- Allow re-using cli auth token - [PR #14780](https://github.com/BerriAI/litellm/pull/14780)
- Create a python method to login using litellm proxy - [PR #14782](https://github.com/BerriAI/litellm/pull/14782)
- Fixes for LiteLLM Proxy CLI to Auth to Gateway - [PR #14836](https://github.com/BerriAI/litellm/pull/14836)
**Virtual Keys**
- Initial support for scheduled key rotations - [PR #14877](https://github.com/BerriAI/litellm/pull/14877)
- Allow scheduling key rotations when creating virtual keys - [PR #14960](https://github.com/BerriAI/litellm/pull/14960)
**Models + Endpoints**
- Fix: added Oracle to provider's list - [PR #14835](https://github.com/BerriAI/litellm/pull/14835)
#### Bugs
- **SSO** - Fix: SSO "Clear" button writes empty values instead of removing SSO config - [PR #14826](https://github.com/BerriAI/litellm/pull/14826)
- **Admin Settings** - Remove useful links from admin settings - [PR #14918](https://github.com/BerriAI/litellm/pull/14918)
- **Management Routes** - Add /user/list to management routes - [PR #14868](https://github.com/BerriAI/litellm/pull/14868)
---
## Logging / Guardrail / Prompt Management Integrations
#### Features
- **[DataDog](../../docs/proxy/logging#datadog)**
- Logging - `datadog` callback Log message content w/o sending to datadog - [PR #14909](https://github.com/BerriAI/litellm/pull/14909)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Adding langfuse usage details for cached tokens - [PR #10955](https://github.com/BerriAI/litellm/pull/10955)
- **[Opik](../../docs/proxy/logging#opik)**
- Improve opik integration code - [PR #14888](https://github.com/BerriAI/litellm/pull/14888)
- **[SQS](../../docs/proxy/logging#sqs)**
- Error logging support for SQS Logger - [PR #14974](https://github.com/BerriAI/litellm/pull/14974)
#### Guardrails
- **LakeraAI v2 Guardrail** - Ensure exception is raised correctly - [PR #14867](https://github.com/BerriAI/litellm/pull/14867)
- **Presidio Guardrail** - Support custom entity types in Presidio guardrail with Union[PiiEntityType, str] - [PR #14899](https://github.com/BerriAI/litellm/pull/14899)
- **Noma Guardrail** - Add noma guardrail provider to ui - [PR #14415](https://github.com/BerriAI/litellm/pull/14415)
#### Prompt Management
- **BitBucket Integration** - Add BitBucket Integration for Prompt Management - [PR #14882](https://github.com/BerriAI/litellm/pull/14882)
---
## Spend Tracking, Budgets and Rate Limiting
- **Service Tier Pricing** - Add service_tier based pricing support for openai (BOTH Service & Priority Support) - [PR #14796](https://github.com/BerriAI/litellm/pull/14796)
- **Cost Tracking** - Show input, output, tool call cost breakdown in StandardLoggingPayload - [PR #14921](https://github.com/BerriAI/litellm/pull/14921)
- **Parallel Request Limiter v3**
- Ensure Lua scripts can execute on redis cluster - [PR #14968](https://github.com/BerriAI/litellm/pull/14968)
- Fix: get metadata info from both metadata and litellm_metadata fields - [PR #14783](https://github.com/BerriAI/litellm/pull/14783)
- **Priority Reservation** - Fix: Priority Reservation: keys without priority metadata receive higher priority than keys with explicit priority configurations - [PR #14832](https://github.com/BerriAI/litellm/pull/14832)
---
## MCP Gateway
- **MCP Configuration** - Enable custom fields in mcp_info configuration - [PR #14794](https://github.com/BerriAI/litellm/pull/14794)
- **MCP Tools** - Remove server_name prefix from list_tools - [PR #14720](https://github.com/BerriAI/litellm/pull/14720)
- **OAuth Flow** - Initial commit for v2 oauth flow - [PR #14964](https://github.com/BerriAI/litellm/pull/14964)
---
## Performance / Loadbalancing / Reliability improvements
- **Memory Leak Fix** - Fix InMemoryCache unbounded growth when TTLs are set - [PR #14869](https://github.com/BerriAI/litellm/pull/14869)
- **Cache Performance** - Fix: cache root cause - [PR #14827](https://github.com/BerriAI/litellm/pull/14827)
- **Concurrency Fix** - Fix concurrency/scaling when many Python threads do streaming using *sync* completions - [PR #14816](https://github.com/BerriAI/litellm/pull/14816)
- **Performance Optimization** - Fix: reduce get_deployment cost to O(1) - [PR #14967](https://github.com/BerriAI/litellm/pull/14967)
- **Performance Optimization** - Fix: remove slow string operation - [PR #14955](https://github.com/BerriAI/litellm/pull/14955)
- **DB Connection Management** - Fix: DB connection state retries - [PR #14925](https://github.com/BerriAI/litellm/pull/14925)
---
## Documentation Updates
- **Provider Documentation** - Fix docs for provider_specific_params.md - [PR #14787](https://github.com/BerriAI/litellm/pull/14787)
- **Model References** - Update model references from gemini-pro to gemini-2.5-pro - [PR #14775](https://github.com/BerriAI/litellm/pull/14775)
- **Letta Guide** - Add Letta Guide documentation - [PR #14798](https://github.com/BerriAI/litellm/pull/14798)
- **README** - Make the README document clearer - [PR #14860](https://github.com/BerriAI/litellm/pull/14860)
- **Session Management** - Update docs for session management availability - [PR #14914](https://github.com/BerriAI/litellm/pull/14914)
- **Cost Documentation** - Add documentation for additional cost-related keys in custom pricing - [PR #14949](https://github.com/BerriAI/litellm/pull/14949)
- **Azure Passthrough** - Add azure passthrough documentation - [PR #14958](https://github.com/BerriAI/litellm/pull/14958)
- **General Documentation** - Doc updates sept 2025 - [PR #14769](https://github.com/BerriAI/litellm/pull/14769)
- Clarified bridging between endpoints and mode in docs.
- Added Vertex AI Gemini API configuration as an alternative in relevant guides.
Linked AWS authentication info in the Bedrock guardrails documentation.
- Added Cancel Response API usage with code snippets
- Clarified that SSO (Single Sign-On) is free for up to 5 users:
- Alphabetized sidebar, leaving quick start / intros at top of categories
- Documented max_connections under cache_params.
- Clarified IAM AssumeRole Policy requirements.
- Added transform utilities example to Getting Started (showing request transformation).
- Added references to models.litellm.ai as the full models list in various docs.
- Added a code snippet for async_post_call_success_hook.
- Removed broken links to callbacks management guide. - Reformatted and linked cookbooks + other relevant docs
- **Documentation Corrections** - Corrected docs updates sept 2025 - [PR #14916](https://github.com/BerriAI/litellm/pull/14916)
---
## New Contributors
* @uzaxirr made their first contribution in [PR #14761](https://github.com/BerriAI/litellm/pull/14761)
* @xprilion made their first contribution in [PR #14416](https://github.com/BerriAI/litellm/pull/14416)
* @CH-GAGANRAJ made their first contribution in [PR #14779](https://github.com/BerriAI/litellm/pull/14779)
* @otaviofbrito made their first contribution in [PR #14778](https://github.com/BerriAI/litellm/pull/14778)
* @danielmklein made their first contribution in [PR #14639](https://github.com/BerriAI/litellm/pull/14639)
* @Jetemple made their first contribution in [PR #14826](https://github.com/BerriAI/litellm/pull/14826)
* @akshoop made their first contribution in [PR #14818](https://github.com/BerriAI/litellm/pull/14818)
* @hazyone made their first contribution in [PR #14821](https://github.com/BerriAI/litellm/pull/14821)
* @leventov made their first contribution in [PR #14816](https://github.com/BerriAI/litellm/pull/14816)
* @fabriciojoc made their first contribution in [PR #10955](https://github.com/BerriAI/litellm/pull/10955)
* @onlylonly made their first contribution in [PR #14845](https://github.com/BerriAI/litellm/pull/14845)
* @Copilot made their first contribution in [PR #14869](https://github.com/BerriAI/litellm/pull/14869)
* @arsh72 made their first contribution in [PR #14899](https://github.com/BerriAI/litellm/pull/14899)
* @berri-teddy made their first contribution in [PR #14914](https://github.com/BerriAI/litellm/pull/14914)
* @vpbill made their first contribution in [PR #14415](https://github.com/BerriAI/litellm/pull/14415)
* @kgritesh made their first contribution in [PR #14893](https://github.com/BerriAI/litellm/pull/14893)
* @oytunkutrup1 made their first contribution in [PR #14858](https://github.com/BerriAI/litellm/pull/14858)
* @nherment made their first contribution in [PR #14933](https://github.com/BerriAI/litellm/pull/14933)
* @deepanshululla made their first contribution in [PR #14974](https://github.com/BerriAI/litellm/pull/14974)
* @TeddyAmkie made their first contribution in [PR #14758](https://github.com/BerriAI/litellm/pull/14758)
* @SmartManoj made their first contribution in [PR #14775](https://github.com/BerriAI/litellm/pull/14775)
* @uc4w6c made their first contribution in [PR #14720](https://github.com/BerriAI/litellm/pull/14720)
* @luizrennocosta made their first contribution in [PR #14783](https://github.com/BerriAI/litellm/pull/14783)
* @AlexsanderHamir made their first contribution in [PR #14827](https://github.com/BerriAI/litellm/pull/14827)
* @dharamendrak made their first contribution in [PR #14721](https://github.com/BerriAI/litellm/pull/14721)
* @TomeHirata made their first contribution in [PR #14164](https://github.com/BerriAI/litellm/pull/14164)
* @mrFranklin made their first contribution in [PR #14860](https://github.com/BerriAI/litellm/pull/14860)
* @luisfucros made their first contribution in [PR #14866](https://github.com/BerriAI/litellm/pull/14866)
* @huangyafei made their first contribution in [PR #14879](https://github.com/BerriAI/litellm/pull/14879)
* @thiswillbeyourgithub made their first contribution in [PR #14949](https://github.com/BerriAI/litellm/pull/14949)
* @Maximgitman made their first contribution in [PR #14965](https://github.com/BerriAI/litellm/pull/14965)
* @subnet-dev made their first contribution in [PR #14938](https://github.com/BerriAI/litellm/pull/14938)
* @22mSqRi made their first contribution in [PR #14972](https://github.com/BerriAI/litellm/pull/14972)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.77.3.rc.1...v1.77.5.rc.1)**

2
docs/my-website/sidebars.js
View file

@ -342,6 +342,7 @@ const sidebars = {
"pass_through/anthropic_completion",
"pass_through/assembly_ai",
"pass_through/bedrock",
"pass_through/azure_passthrough",
"pass_through/cohere",
"pass_through/google_ai_studio",
"pass_through/langfuse",
@ -583,6 +584,7 @@ const sidebars = {
"budget_manager",
"caching/all_caches",
"completion/token_usage",
"sdk_custom_pricing",
"embedding/async_embedding",
"embedding/moderation",
"migration",

7
enterprise/litellm_enterprise/integrations/prometheus.py
View file

@ -2262,9 +2262,12 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
keys_parts = key.split(".")
# Traverse through the dictionary using the parts
value = metadata
value: Any = metadata
for part in keys_parts:
value = value.get(part, None) # Get the value, return None if not found
if isinstance(value, dict):
value = value.get(part, None) # Get the value, return None if not found
else:
value = None
if value is None:
break

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.21-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.21.tar.gz vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.22-py3-none-any.whl vendored Normal file
View file

Binary file not shown.

BIN
litellm-proxy-extras/dist/litellm_proxy_extras-0.2.22.tar.gz vendored Normal file
View file

Binary file not shown.

7
litellm-proxy-extras/litellm_proxy_extras/migrations/20250926194702_unnamed_migration/migration.sql Normal file
View file

@ -0,0 +1,7 @@
-- AlterTable
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN "auto_rotate" BOOLEAN DEFAULT false,
ADD COLUMN "key_rotation_at" TIMESTAMP(3),
ADD COLUMN "last_rotation_at" TIMESTAMP(3),
ADD COLUMN "rotation_count" INTEGER DEFAULT 0,
ADD COLUMN "rotation_interval" TEXT;

1
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -225,6 +225,7 @@ model LiteLLM_VerificationToken {
auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated
rotation_interval String? // How often to rotate (e.g., "30d", "90d")
last_rotation_at DateTime? // When this key was last rotated
key_rotation_at DateTime? // When this key should next be rotated
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])

50
litellm-proxy-extras/migration_runbook.md Normal file
View file

@ -0,0 +1,50 @@
# Database Migration Runbook
This is a runbook for creating and running database migrations for the LiteLLM proxy. For use for litellm engineers only.
## Quick Start
```bash
# Install deps (one time)
pip install testing.postgresql
brew install postgresql@14 # macOS
# Add to PATH
export PATH="/opt/homebrew/opt/postgresql@14/bin:$PATH"
# Run migration
python ci_cd/run_migration.py "your_migration_name"
```
## What It Does
1. Creates temp PostgreSQL DB
2. Applies existing migrations
3. Compares with `schema.prisma`
4. Generates new migration if changes found
## Common Fixes
**Missing testing module:**
```bash
pip install testing.postgresql
```
**initdb not found:**
```bash
brew install postgresql@14
export PATH="/opt/homebrew/opt/postgresql@14/bin:$PATH"
```
**Empty migration directory error:**
```bash
rm -rf litellm-proxy-extras/litellm_proxy_extras/migrations/[empty_dir]
```
## Rules
- Update `schema.prisma` first
- Review generated SQL before committing
- Use descriptive migration names
- Never edit existing migration files
- Commit schema + migration together

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.2.20"
version = "0.2.22"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.2.20"
version = "0.2.22"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",

9
litellm/__init__.py
View file

@ -377,7 +377,9 @@ public_model_groups: Optional[List[str]] = None
public_model_groups_links: Dict[str, str] = {}
#### REQUEST PRIORITIZATION ######
priority_reservation: Optional[Dict[str, float]] = None
priority_reservation_settings: "PriorityReservationSettings" = PriorityReservationSettings()
priority_reservation_settings: "PriorityReservationSettings" = (
PriorityReservationSettings()
)
######## Networking Settings ########
@ -443,7 +445,7 @@ def identify(event_details):
####### ADDITIONAL PARAMS ################### configurable params if you use proxy models like Helicone, map spend to org id, etc.
api_base: Optional[str] = None
headers = None
api_version = None
api_version: Optional[str] = None
organization = None
project = None
config_path = None
@ -494,7 +496,7 @@ azure_ai_models: Set = set()
jina_ai_models: Set = set()
voyage_models: Set = set()
infinity_models: Set = set()
heroku_models: Set = set()
heroku_models: Set = set()
databricks_models: Set = set()
cloudflare_models: Set = set()
codestral_models: Set = set()
@ -1357,6 +1359,7 @@ from .passthrough import allm_passthrough_route, llm_passthrough_route
### GLOBAL CONFIG ###
global_bitbucket_config: Optional[Dict[str, Any]] = None
def set_global_bitbucket_config(config: Dict[str, Any]) -> None:
"""Set global BitBucket configuration for prompt management."""
global global_bitbucket_config

17
litellm/experimental_mcp_client/client.py
View file

@ -1,10 +1,11 @@
"""
LiteLLM Proxy uses this MCP Client to connnect to other MCP servers.
"""
import asyncio
import base64
from datetime import timedelta
from typing import List, Optional
from typing import Dict, List, Optional
from mcp import ClientSession, StdioServerParameters
from mcp.client.sse import sse_client
@ -46,6 +47,7 @@ class MCPClient:
auth_value: Optional[str] = None,
timeout: float = 60.0,
stdio_config: Optional[MCPStdioConfig] = None,
extra_headers: Optional[Dict[str, str]] = None,
):
self.server_url: str = server_url
self.transport_type: MCPTransport = transport_type
@ -59,7 +61,7 @@ class MCPClient:
self._session_ctx = None
self._task: Optional[asyncio.Task] = None
self.stdio_config: Optional[MCPStdioConfig] = stdio_config
self.extra_headers: Optional[Dict[str, str]] = extra_headers
# handle the basic auth value if provided
if auth_value:
self.update_auth_value(auth_value)
@ -115,6 +117,9 @@ class MCPClient:
await self._session.initialize()
else: # http
headers = self._get_auth_headers()
verbose_logger.debug(
"litellm headers for streamablehttp_client: ", headers
)
self._transport_ctx = streamablehttp_client(
url=self.server_url,
timeout=timedelta(seconds=self.timeout),
@ -186,9 +191,7 @@ class MCPClient:
def _get_auth_headers(self) -> dict:
"""Generate authentication headers based on auth type."""
headers = {
"MCP-Protocol-Version": "2025-06-18"
}
headers = {"MCP-Protocol-Version": "2025-06-18"}
if self._mcp_auth_value:
if self.auth_type == MCPAuth.bearer_token:
@ -200,6 +203,10 @@ class MCPClient:
elif self.auth_type == MCPAuth.authorization:
headers["Authorization"] = self._mcp_auth_value
# update the headers with the extra headers
if self.extra_headers:
headers.update(self.extra_headers)
return headers
async def list_tools(self) -> List[MCPTool]:

22
litellm/integrations/bitbucket/bitbucket_prompt_manager.py
View file

@ -116,7 +116,7 @@ class BitBucketTemplateManager:
template_content = content
# Parse YAML frontmatter
metadata = {}
metadata: Dict[str, Any] = {}
if frontmatter_str:
try:
import yaml
@ -136,7 +136,7 @@ class BitBucketTemplateManager:
def _parse_yaml_basic(self, yaml_str: str) -> Dict[str, Any]:
"""Basic YAML parser for simple cases when PyYAML is not available."""
result = {}
result: Dict[str, Any] = {}
for line in yaml_str.split("\n"):
line = line.strip()
if ":" in line and not line.startswith("#"):
@ -156,7 +156,7 @@ class BitBucketTemplateManager:
return result
def render_template(
self, template_id: str, variables: Dict[str, Any] = None
self, template_id: str, variables: Optional[Dict[str, Any]] = None
) -> str:
"""Render a template with the given variables."""
if template_id not in self.prompts:
@ -209,7 +209,7 @@ class BitBucketPromptManager(CustomPromptManagement):
):
self.bitbucket_config = bitbucket_config
self.prompt_id = prompt_id
self._prompt_manager: Optional[BitBucketPromptManager] = None
self._prompt_manager: Optional[BitBucketTemplateManager] = None
@property
def integration_name(self) -> str:
@ -288,11 +288,11 @@ class BitBucketPromptManager(CustomPromptManagement):
# Merge with existing messages
if parsed_messages:
# If we have parsed messages, use them instead of the original messages
final_messages = parsed_messages
final_messages: List[AllMessageValues] = parsed_messages
else:
# If no messages were parsed, prepend the prompt to existing messages
final_messages = [
{"role": "user", "content": rendered_prompt}
{"role": "user", "content": rendered_prompt} # type: ignore
] + messages
# Update litellm_params with prompt metadata
@ -346,7 +346,7 @@ class BitBucketPromptManager(CustomPromptManagement):
{
"role": current_role,
"content": "\n".join(current_content).strip(),
}
} # type: ignore
)
current_role = "system"
current_content = [line[7:].strip()] # Remove "System:" prefix
@ -356,7 +356,7 @@ class BitBucketPromptManager(CustomPromptManagement):
{
"role": current_role,
"content": "\n".join(current_content).strip(),
}
} # type: ignore
)
current_role = "user"
current_content = [line[5:].strip()] # Remove "User:" prefix
@ -366,7 +366,7 @@ class BitBucketPromptManager(CustomPromptManagement):
{
"role": current_role,
"content": "\n".join(current_content).strip(),
}
} # type: ignore
)
current_role = "assistant"
current_content = [line[10:].strip()] # Remove "Assistant:" prefix
@ -382,9 +382,9 @@ class BitBucketPromptManager(CustomPromptManagement):
# If no role indicators found, treat as a single user message
if not messages and prompt_content.strip():
messages = [{"role": "user", "content": prompt_content.strip()}]
messages = [{"role": "user", "content": prompt_content.strip()}] # type: ignore
return messages
return messages # type: ignore
def post_call_hook(
self,

20
litellm/integrations/sqs.py
View file

@ -7,6 +7,7 @@ This logger sends ``StandardLoggingPayload`` entries to an AWS SQS queue.
from __future__ import annotations
import asyncio
import traceback
from typing import List, Optional
import litellm
@ -200,6 +201,25 @@ class SQSLogger(CustomBatchLogger, BaseAWSLLM):
except Exception as e:
verbose_logger.exception(f"sqs Layer Error - {str(e)}")
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
try:
standard_logging_payload = kwargs.get("standard_logging_object")
if standard_logging_payload is None:
raise ValueError("standard_logging_payload is None")
self.log_queue.append(standard_logging_payload)
verbose_logger.debug(
"sqs logging: queue length %s, batch size %s",
len(self.log_queue),
self.batch_size,
)
except Exception as e:
verbose_logger.exception(
f"Datadog Layer Error - {str(e)}\n{traceback.format_exc()}"
)
pass
async def async_send_batch(self) -> None:
verbose_logger.debug(
f"sqs logger - sending batch of {len(self.log_queue)}"

2
litellm/litellm_core_utils/custom_logger_registry.py
View file

@ -15,6 +15,7 @@ from litellm.integrations.agentops import AgentOps
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.integrations.argilla import ArgillaLogger
from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger
from litellm.integrations.bitbucket import BitBucketPromptManager
from litellm.integrations.braintrust_logging import BraintrustLogger
from litellm.integrations.datadog.datadog import DataDogLogger
from litellm.integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger
@ -39,7 +40,6 @@ try:
from litellm_enterprise.integrations.prometheus import PrometheusLogger
except Exception:
PrometheusLogger = None
from litellm.integrations.bitbucket import BitBucketPromptManager
from litellm.integrations.cloudzero.cloudzero import CloudZeroLogger
from litellm.integrations.dotprompt import DotpromptManager
from litellm.integrations.s3_v2 import S3Logger

325
litellm/litellm_core_utils/litellm_logging.py
View file

@ -301,9 +301,9 @@ class Logging(LiteLLMLoggingBaseClass):
self.litellm_trace_id: str = litellm_trace_id or str(uuid.uuid4())
self.function_id = function_id
self.streaming_chunks: List[Any] = [] # for generating complete stream response
self.sync_streaming_chunks: List[
Any
] = [] # for generating complete stream response
self.sync_streaming_chunks: List[Any] = (
[]
) # for generating complete stream response
self.log_raw_request_response = log_raw_request_response
# Initialize dynamic callbacks
@ -344,7 +344,7 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_params = scrub_sensitive_keys_in_metadata(litellm_params)
self.litellm_params = litellm_params
# Initialize cost breakdown field
self.cost_breakdown: Optional[CostBreakdown] = None
@ -676,9 +676,9 @@ class Logging(LiteLLMLoggingBaseClass):
if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
non_default_params
):
self.model_call_details[
"prompt_integration"
] = anthropic_cache_control_logger.__class__.__name__
self.model_call_details["prompt_integration"] = (
anthropic_cache_control_logger.__class__.__name__
)
return anthropic_cache_control_logger
#########################################################
@ -690,9 +690,9 @@ class Logging(LiteLLMLoggingBaseClass):
internal_usage_cache=None,
llm_router=None,
)
self.model_call_details[
"prompt_integration"
] = vector_store_custom_logger.__class__.__name__
self.model_call_details["prompt_integration"] = (
vector_store_custom_logger.__class__.__name__
)
return vector_store_custom_logger
return None
@ -744,9 +744,9 @@ class Logging(LiteLLMLoggingBaseClass):
model
): # if model name was changes pre-call, overwrite the initial model call name with the new one
self.model_call_details["model"] = model
self.model_call_details["litellm_params"][
"api_base"
] = self._get_masked_api_base(additional_args.get("api_base", ""))
self.model_call_details["litellm_params"]["api_base"] = (
self._get_masked_api_base(additional_args.get("api_base", ""))
)
def pre_call(self, input, api_key, model=None, additional_args={}): # noqa: PLR0915
# Log the exact input to the LLM API
@ -775,10 +775,10 @@ class Logging(LiteLLMLoggingBaseClass):
try:
# [Non-blocking Extra Debug Information in metadata]
if turn_off_message_logging is True:
_metadata[
"raw_request"
] = "redacted by litellm. \
_metadata["raw_request"] = (
"redacted by litellm. \
'litellm.turn_off_message_logging=True'"
)
else:
curl_command = self._get_request_curl_command(
api_base=additional_args.get("api_base", ""),
@ -789,32 +789,32 @@ class Logging(LiteLLMLoggingBaseClass):
_metadata["raw_request"] = str(curl_command)
# split up, so it's easier to parse in the UI
self.model_call_details[
"raw_request_typed_dict"
] = RawRequestTypedDict(
raw_request_api_base=str(
additional_args.get("api_base") or ""
),
raw_request_body=self._get_raw_request_body(
additional_args.get("complete_input_dict", {})
),
raw_request_headers=self._get_masked_headers(
additional_args.get("headers", {}) or {},
ignore_sensitive_headers=True,
),
error=None,
self.model_call_details["raw_request_typed_dict"] = (
RawRequestTypedDict(
raw_request_api_base=str(
additional_args.get("api_base") or ""
),
raw_request_body=self._get_raw_request_body(
additional_args.get("complete_input_dict", {})
),
raw_request_headers=self._get_masked_headers(
additional_args.get("headers", {}) or {},
ignore_sensitive_headers=True,
),
error=None,
)
)
except Exception as e:
self.model_call_details[
"raw_request_typed_dict"
] = RawRequestTypedDict(
error=str(e),
self.model_call_details["raw_request_typed_dict"] = (
RawRequestTypedDict(
error=str(e),
)
)
_metadata[
"raw_request"
] = "Unable to Log \
_metadata["raw_request"] = (
"Unable to Log \
raw request: {}".format(
str(e)
str(e)
)
)
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
@ -1115,13 +1115,13 @@ class Logging(LiteLLMLoggingBaseClass):
for callback in callbacks:
try:
if isinstance(callback, CustomLogger):
response: Optional[
MCPPostCallResponseObject
] = await callback.async_post_mcp_tool_call_hook(
kwargs=kwargs,
response_obj=post_mcp_tool_call_response_obj,
start_time=start_time,
end_time=end_time,
response: Optional[MCPPostCallResponseObject] = (
await callback.async_post_mcp_tool_call_hook(
kwargs=kwargs,
response_obj=post_mcp_tool_call_response_obj,
start_time=start_time,
end_time=end_time,
)
)
######################################################################
# if any of the callbacks modify the response, use the modified response
@ -1168,19 +1168,19 @@ class Logging(LiteLLMLoggingBaseClass):
) -> None:
"""
Helper method to store cost breakdown in the logging object.
Args:
input_cost: Cost of input/prompt tokens
output_cost: Cost of output/completion tokens
output_cost: Cost of output/completion tokens
cost_for_built_in_tools_cost_usd_dollar: Cost of built-in tools
total_cost: Total cost of request
"""
self.cost_breakdown = CostBreakdown(
input_cost=input_cost,
output_cost=output_cost,
total_cost=total_cost,
tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar
tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar,
)
verbose_logger.debug(
f"Cost breakdown set - input: {input_cost}, output: {output_cost}, cost_for_built_in_tools_cost_usd_dollar: {cost_for_built_in_tools_cost_usd_dollar}, total: {total_cost}"
@ -1259,9 +1259,11 @@ class Logging(LiteLLMLoggingBaseClass):
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,
"litellm_logging_obj": self,
"service_tier": self.optional_params.get("service_tier")
if self.optional_params
else None,
"service_tier": (
self.optional_params.get("service_tier")
if self.optional_params
else None
),
}
except Exception as e: # error creating kwargs for cost calculation
debug_info = StandardLoggingModelCostFailureDebugInformation(
@ -1271,9 +1273,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
f"response_cost_failure_debug_information: {debug_info}"
)
self.model_call_details[
"response_cost_failure_debug_information"
] = debug_info
self.model_call_details["response_cost_failure_debug_information"] = (
debug_info
)
return None
try:
@ -1298,9 +1300,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
f"response_cost_failure_debug_information: {debug_info}"
)
self.model_call_details[
"response_cost_failure_debug_information"
] = debug_info
self.model_call_details["response_cost_failure_debug_information"] = (
debug_info
)
return None
@ -1444,9 +1446,9 @@ class Logging(LiteLLMLoggingBaseClass):
end_time = datetime.datetime.now()
if self.completion_start_time is None:
self.completion_start_time = end_time
self.model_call_details[
"completion_start_time"
] = self.completion_start_time
self.model_call_details["completion_start_time"] = (
self.completion_start_time
)
self.model_call_details["log_event_type"] = "successful_api_call"
self.model_call_details["end_time"] = end_time
self.model_call_details["cache_hit"] = cache_hit
@ -1499,39 +1501,39 @@ class Logging(LiteLLMLoggingBaseClass):
"response_cost"
]
else:
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(result=logging_result)
self.model_call_details["response_cost"] = (
self._response_cost_calculator(result=logging_result)
)
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
elif isinstance(result, dict) or isinstance(result, list):
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
elif standard_logging_object is not None:
self.model_call_details[
"standard_logging_object"
] = standard_logging_object
self.model_call_details["standard_logging_object"] = (
standard_logging_object
)
else: # streaming chunks + image gen.
self.model_call_details["response_cost"] = None
@ -1682,23 +1684,23 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
"Logging Details LiteLLM-Success Call streaming complete"
)
self.model_call_details[
"complete_streaming_response"
] = complete_streaming_response
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(result=complete_streaming_response)
self.model_call_details["complete_streaming_response"] = (
complete_streaming_response
)
self.model_call_details["response_cost"] = (
self._response_cost_calculator(result=complete_streaming_response)
)
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
callbacks = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_success_callbacks,
@ -2026,10 +2028,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
else:
if self.stream and complete_streaming_response:
self.model_call_details[
"complete_response"
] = self.model_call_details.get(
"complete_streaming_response", {}
self.model_call_details["complete_response"] = (
self.model_call_details.get(
"complete_streaming_response", {}
)
)
result = self.model_call_details["complete_response"]
openMeterLogger.log_success_event(
@ -2068,10 +2070,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
else:
if self.stream and complete_streaming_response:
self.model_call_details[
"complete_response"
] = self.model_call_details.get(
"complete_streaming_response", {}
self.model_call_details["complete_response"] = (
self.model_call_details.get(
"complete_streaming_response", {}
)
)
result = self.model_call_details["complete_response"]
@ -2209,9 +2211,9 @@ class Logging(LiteLLMLoggingBaseClass):
if complete_streaming_response is not None:
print_verbose("Async success callbacks: Got a complete streaming response")
self.model_call_details[
"async_complete_streaming_response"
] = complete_streaming_response
self.model_call_details["async_complete_streaming_response"] = (
complete_streaming_response
)
try:
if self.model_call_details.get("cache_hit", False) is True:
@ -2222,10 +2224,10 @@ class Logging(LiteLLMLoggingBaseClass):
model_call_details=self.model_call_details
)
# base_model defaults to None if not set on model_info
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(
result=complete_streaming_response
self.model_call_details["response_cost"] = (
self._response_cost_calculator(
result=complete_streaming_response
)
)
verbose_logger.debug(
@ -2238,16 +2240,16 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = None
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
callbacks = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_async_success_callbacks,
@ -2460,18 +2462,18 @@ class Logging(LiteLLMLoggingBaseClass):
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj={},
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="failure",
error_str=str(exception),
original_exception=exception,
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj={},
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="failure",
error_str=str(exception),
original_exception=exception,
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
return start_time, end_time
@ -2979,14 +2981,17 @@ class Logging(LiteLLMLoggingBaseClass):
- For Non-streaming responses, we need to transform the response to a ModelResponse object.
- For streaming responses, anthropic_messages handler calls success_handler with a assembled ModelResponse.
"""
import httpx
if self.stream and isinstance(result, ModelResponse):
return result
elif isinstance(result, ModelResponse):
return result
if "httpx_response" in self.model_call_details:
httpx_response = self.model_call_details.get("httpx_response", None)
if httpx_response and isinstance(httpx_response, httpx.Response):
result = litellm.AnthropicConfig().transform_response(
raw_response=self.model_call_details.get("httpx_response", None),
raw_response=httpx_response,
model_response=litellm.ModelResponse(),
model=self.model,
messages=[],
@ -3355,9 +3360,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
endpoint=arize_config.endpoint,
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = f"space_id={arize_config.space_key},api_key={arize_config.api_key}"
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
f"space_id={arize_config.space_key},api_key={arize_config.api_key}"
)
for callback in _in_memory_loggers:
if (
isinstance(callback, ArizeLogger)
@ -3381,9 +3386,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
# auth can be disabled on local deployments of arize phoenix
if arize_phoenix_config.otlp_auth_headers is not None:
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = arize_phoenix_config.otlp_auth_headers
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
arize_phoenix_config.otlp_auth_headers
)
for callback in _in_memory_loggers:
if (
@ -3515,9 +3520,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
exporter="otlp_http",
endpoint="https://langtrace.ai/api/trace",
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
f"api_key={os.getenv('LANGTRACE_API_KEY')}"
)
for callback in _in_memory_loggers:
if (
isinstance(callback, OpenTelemetry)
@ -4197,10 +4202,10 @@ class StandardLoggingPayloadSetup:
for key in StandardLoggingHiddenParams.__annotations__.keys():
if key in hidden_params:
if key == "additional_headers":
clean_hidden_params[
"additional_headers"
] = StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
clean_hidden_params["additional_headers"] = (
StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
)
)
else:
clean_hidden_params[key] = hidden_params[key] # type: ignore
@ -4252,9 +4257,9 @@ class StandardLoggingPayloadSetup:
if (
custom_logger
and hasattr(custom_logger, "s3_path")
and custom_logger.s3_path
and getattr(custom_logger, "s3_path")
):
s3_path = custom_logger.s3_path
s3_path = getattr(custom_logger, "s3_path")
except Exception:
# If any error occurs in getting the logger instance, use default empty s3_path
pass
@ -4704,9 +4709,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
):
for k, v in metadata["user_api_key_metadata"].items():
if k == "logging": # prevent logging user logging keys
cleaned_user_api_key_metadata[
k
] = "scrubbed_by_litellm_for_sensitive_keys"
cleaned_user_api_key_metadata[k] = (
"scrubbed_by_litellm_for_sensitive_keys"
)
else:
cleaned_user_api_key_metadata[k] = v

171
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -47,7 +47,7 @@ class StandardBuiltInToolCostTracking:
- Code Interpreter (Azure)
"""
standard_built_in_tools_params = standard_built_in_tools_params or {}
# Handle web search
if StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
response_object=response_object, usage=usage
@ -58,7 +58,7 @@ class StandardBuiltInToolCostTracking:
usage=usage,
standard_built_in_tools_params=standard_built_in_tools_params,
)
# Handle file search
if StandardBuiltInToolCostTracking.response_object_includes_file_search_call(
response_object=response_object
@ -68,7 +68,7 @@ class StandardBuiltInToolCostTracking:
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
# Handle Azure assistant features
return StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
model=model,
@ -85,14 +85,14 @@ class StandardBuiltInToolCostTracking:
) -> float:
"""Handle web search cost calculation."""
from litellm.llms import get_cost_for_web_search_request
model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
if custom_llm_provider is None and model_info is not None:
custom_llm_provider = model_info["litellm_provider"]
if (
model_info is not None
and usage is not None
@ -105,9 +105,11 @@ class StandardBuiltInToolCostTracking:
)
if result is not None:
return result
return StandardBuiltInToolCostTracking.get_cost_for_web_search(
web_search_options=standard_built_in_tools_params.get("web_search_options", None),
web_search_options=standard_built_in_tools_params.get(
"web_search_options", None
),
model_info=model_info,
)
@ -121,12 +123,17 @@ class StandardBuiltInToolCostTracking:
model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
file_search_usage = standard_built_in_tools_params.get("file_search", {})
file_search_raw: Any = standard_built_in_tools_params.get("file_search", {})
file_search_usage: Optional[FileSearchTool] = (
FileSearchTool(**file_search_raw) if file_search_raw else None
)
# Convert model_info to dict and extract usage parameters
model_info_dict = dict(model_info) if model_info is not None else None
storage_gb, days = StandardBuiltInToolCostTracking._extract_file_search_params(file_search_usage)
storage_gb, days = StandardBuiltInToolCostTracking._extract_file_search_params(
file_search_usage
)
return StandardBuiltInToolCostTracking.get_cost_for_file_search(
file_search=file_search_usage,
provider=custom_llm_provider,
@ -144,11 +151,11 @@ class StandardBuiltInToolCostTracking:
"""Handle Azure assistant features cost calculation."""
if custom_llm_provider != "azure":
return 0.0
model_info = StandardBuiltInToolCostTracking._safe_get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
total_cost = 0.0
total_cost += StandardBuiltInToolCostTracking._get_vector_store_cost(
model_info, custom_llm_provider, standard_built_in_tools_params
@ -159,31 +166,33 @@ class StandardBuiltInToolCostTracking:
total_cost += StandardBuiltInToolCostTracking._get_code_interpreter_cost(
model_info, custom_llm_provider, standard_built_in_tools_params
)
return total_cost
@staticmethod
def _extract_file_search_params(file_search_usage: Any) -> Tuple[Optional[float], Optional[float]]:
def _extract_file_search_params(
file_search_usage: Any,
) -> Tuple[Optional[float], Optional[float]]:
"""Extract and convert file search parameters safely."""
storage_gb = None
days = None
if isinstance(file_search_usage, dict):
storage_gb_val = file_search_usage.get("storage_gb")
days_val = file_search_usage.get("days")
if storage_gb_val is not None:
try:
storage_gb = float(storage_gb_val) # type: ignore
except (TypeError, ValueError):
storage_gb = None
if days_val is not None:
try:
days = float(days_val) # type: ignore
except (TypeError, ValueError):
days = None
return storage_gb, days
@staticmethod
@ -193,13 +202,17 @@ class StandardBuiltInToolCostTracking:
standard_built_in_tools_params: StandardBuiltInToolsParams,
) -> float:
"""Calculate vector store cost."""
vector_store_usage = standard_built_in_tools_params.get("vector_store_usage", None)
vector_store_usage = standard_built_in_tools_params.get(
"vector_store_usage", None
)
if not vector_store_usage:
return 0.0
model_info_dict = dict(model_info) if model_info is not None else None
vector_store_dict = vector_store_usage if isinstance(vector_store_usage, dict) else {}
vector_store_dict = (
vector_store_usage if isinstance(vector_store_usage, dict) else {}
)
return StandardBuiltInToolCostTracking.get_cost_for_vector_store(
vector_store_usage=vector_store_dict,
provider=custom_llm_provider,
@ -213,13 +226,17 @@ class StandardBuiltInToolCostTracking:
standard_built_in_tools_params: StandardBuiltInToolsParams,
) -> float:
"""Calculate computer use cost."""
computer_use_usage = standard_built_in_tools_params.get("computer_use_usage", {})
computer_use_usage = standard_built_in_tools_params.get(
"computer_use_usage", {}
)
if not computer_use_usage:
return 0.0
model_info_dict = dict(model_info) if model_info is not None else None
input_tokens, output_tokens = StandardBuiltInToolCostTracking._extract_token_counts(computer_use_usage)
input_tokens, output_tokens = (
StandardBuiltInToolCostTracking._extract_token_counts(computer_use_usage)
)
return StandardBuiltInToolCostTracking.get_cost_for_computer_use(
input_tokens=input_tokens,
output_tokens=output_tokens,
@ -234,13 +251,17 @@ class StandardBuiltInToolCostTracking:
standard_built_in_tools_params: StandardBuiltInToolsParams,
) -> float:
"""Calculate code interpreter cost."""
code_interpreter_sessions = standard_built_in_tools_params.get("code_interpreter_sessions", None)
code_interpreter_sessions = standard_built_in_tools_params.get(
"code_interpreter_sessions", None
)
if not code_interpreter_sessions:
return 0.0
model_info_dict = dict(model_info) if model_info is not None else None
sessions = StandardBuiltInToolCostTracking._safe_convert_to_int(code_interpreter_sessions)
sessions = StandardBuiltInToolCostTracking._safe_convert_to_int(
code_interpreter_sessions
)
return StandardBuiltInToolCostTracking.get_cost_for_code_interpreter(
sessions=sessions,
provider=custom_llm_provider,
@ -248,18 +269,24 @@ class StandardBuiltInToolCostTracking:
)
@staticmethod
def _extract_token_counts(computer_use_usage: Any) -> Tuple[Optional[int], Optional[int]]:
def _extract_token_counts(
computer_use_usage: Any,
) -> Tuple[Optional[int], Optional[int]]:
"""Extract and convert token counts safely."""
input_tokens = None
output_tokens = None
if isinstance(computer_use_usage, dict):
input_tokens_val = computer_use_usage.get("input_tokens")
output_tokens_val = computer_use_usage.get("output_tokens")
input_tokens = StandardBuiltInToolCostTracking._safe_convert_to_int(input_tokens_val)
output_tokens = StandardBuiltInToolCostTracking._safe_convert_to_int(output_tokens_val)
input_tokens = StandardBuiltInToolCostTracking._safe_convert_to_int(
input_tokens_val
)
output_tokens = StandardBuiltInToolCostTracking._safe_convert_to_int(
output_tokens_val
)
return input_tokens, output_tokens
@staticmethod
@ -400,8 +427,11 @@ class StandardBuiltInToolCostTracking:
if model_info is None:
return 0.0
search_context_raw: Any = model_info.get("search_context_cost_per_query", {})
search_context_pricing: SearchContextCostPerQuery = (
model_info.get("search_context_cost_per_query", {}) or {}
SearchContextCostPerQuery(**search_context_raw)
if search_context_raw
else SearchContextCostPerQuery()
)
if web_search_options.get("search_context_size", None) == "low":
return search_context_pricing.get("search_context_size_low", 0.0)
@ -424,9 +454,12 @@ class StandardBuiltInToolCostTracking:
"""
if model_info is None:
return 0.0
search_context_raw: Any = model_info.get("search_context_cost_per_query", {}) or {}
search_context_pricing: SearchContextCostPerQuery = (
model_info.get("search_context_cost_per_query", {}) or {}
) or {}
SearchContextCostPerQuery(**search_context_raw)
if search_context_raw
else SearchContextCostPerQuery()
)
return search_context_pricing.get("search_context_size_medium", 0.0)
@staticmethod
@ -445,22 +478,27 @@ class StandardBuiltInToolCostTracking:
"""
if file_search is None:
return 0.0
# Check if model-specific pricing is available
if model_info and "file_search_cost_per_gb_per_day" in model_info and provider == "azure":
if (
model_info
and "file_search_cost_per_gb_per_day" in model_info
and provider == "azure"
):
if storage_gb and days:
return storage_gb * days * model_info["file_search_cost_per_gb_per_day"]
elif model_info and "file_search_cost_per_1k_calls" in model_info:
return model_info["file_search_cost_per_1k_calls"]
# Azure has storage-based pricing for file search
if provider == "azure":
from litellm.constants import AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY
if storage_gb and days:
return storage_gb * days * AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY
# Default to 0 if no storage info provided
return 0.0
# Default to OpenAI pricing (per-call based)
return OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
@ -472,24 +510,25 @@ class StandardBuiltInToolCostTracking:
) -> float:
"""
Calculate cost for vector store usage.
Azure charges based on storage size and duration.
"""
if vector_store_usage is None:
return 0.0
storage_gb = vector_store_usage.get("storage_gb", 0.0)
days = vector_store_usage.get("days", 0.0)
# Check if model-specific pricing is available
if model_info and "vector_store_cost_per_gb_per_day" in model_info:
return storage_gb * days * model_info["vector_store_cost_per_gb_per_day"]
# Azure has different pricing structure for vector store
if provider == "azure":
from litellm.constants import AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY
return storage_gb * days * AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY
# OpenAI doesn't charge separately for vector store (included in embeddings)
return 0.0
@ -502,14 +541,18 @@ class StandardBuiltInToolCostTracking:
) -> float:
"""
Calculate cost for computer use feature.
Azure: $0.003 USD per 1K input tokens, $0.012 USD per 1K output tokens
"""
if provider == "azure" and (input_tokens or output_tokens):
# Check if model-specific pricing is available
if model_info:
input_cost = model_info.get("computer_use_input_cost_per_1k_tokens", 0.0)
output_cost = model_info.get("computer_use_output_cost_per_1k_tokens", 0.0)
input_cost = model_info.get(
"computer_use_input_cost_per_1k_tokens", 0.0
)
output_cost = model_info.get(
"computer_use_output_cost_per_1k_tokens", 0.0
)
if input_cost or output_cost:
total_cost = 0.0
if input_tokens:
@ -517,19 +560,24 @@ class StandardBuiltInToolCostTracking:
if output_tokens:
total_cost += (output_tokens / 1000.0) * output_cost
return total_cost
# Azure default pricing
from litellm.constants import (
AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS,
AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS,
)
total_cost = 0.0
if input_tokens:
total_cost += (input_tokens / 1000.0) * AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS
total_cost += (
input_tokens / 1000.0
) * AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS
if output_tokens:
total_cost += (output_tokens / 1000.0) * AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS
total_cost += (
output_tokens / 1000.0
) * AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS
return total_cost
# OpenAI doesn't charge separately for computer use yet
return 0.0
@ -541,21 +589,22 @@ class StandardBuiltInToolCostTracking:
) -> float:
"""
Calculate cost for code interpreter feature.
Azure: $0.03 USD per session
"""
if sessions is None or sessions == 0:
return 0.0
# Check if model-specific pricing is available
if model_info and "code_interpreter_cost_per_session" in model_info:
return sessions * model_info["code_interpreter_cost_per_session"]
# Azure pricing for code interpreter
if provider == "azure":
from litellm.constants import AZURE_CODE_INTERPRETER_COST_PER_SESSION
return sessions * AZURE_CODE_INTERPRETER_COST_PER_SESSION
# OpenAI doesn't charge separately for code interpreter yet
return 0.0

15
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -2,11 +2,11 @@ import asyncio
import json
import time
import traceback
from litellm._uuid import uuid
from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_extract_reasoning_content,
@ -31,6 +31,7 @@ from litellm.types.utils import Logprobs as TextCompletionLogprobs
from litellm.types.utils import (
Message,
ModelResponse,
ModelResponseStream,
RerankResponse,
StreamingChoices,
TextChoices,
@ -108,12 +109,12 @@ async def convert_to_streaming_response_async(response_object: Optional[dict] =
if response_object is None:
raise Exception("Error in response object format")
model_response_object = ModelResponse(stream=True)
model_response_object = ModelResponseStream()
if model_response_object is None:
raise Exception("Error in response creating model response object")
choice_list = []
choice_list: List[StreamingChoices] = []
for idx, choice in enumerate(response_object["choices"]):
if (
@ -182,8 +183,8 @@ def convert_to_streaming_response(response_object: Optional[dict] = None):
if response_object is None:
raise Exception("Error in response object format")
model_response_object = ModelResponse(stream=True)
choice_list = []
model_response_object = ModelResponseStream()
choice_list: List[StreamingChoices] = []
for idx, choice in enumerate(response_object["choices"]):
delta = Delta(**choice["message"])
finish_reason = choice.get("finish_reason", None)
@ -460,7 +461,7 @@ def convert_to_model_response_object( # noqa: PLR0915
if stream is True:
# for returning cached responses, we need to yield a generator
return convert_to_streaming_response(response_object=response_object)
choice_list = []
choice_list: List[Choices] = []
assert response_object["choices"] is not None and isinstance(
response_object["choices"], Iterable
@ -564,7 +565,7 @@ def convert_to_model_response_object( # noqa: PLR0915
provider_specific_fields=provider_specific_fields,
)
choice_list.append(choice)
model_response_object.choices = choice_list
model_response_object.choices = choice_list # type: ignore
if "usage" in response_object and response_object["usage"] is not None:
usage_object = litellm.Usage(**response_object["usage"])

5
litellm/litellm_core_utils/rules.py
View file

@ -23,6 +23,11 @@ class Rules:
def __init__(self) -> None:
pass
@staticmethod
def has_pre_call_rules() -> bool:
"""Check if any pre-call rules are configured"""
return len(litellm.pre_call_rules) > 0
def pre_call_rules(self, input: str, model: str):
for rule in litellm.pre_call_rules:
if callable(rule):

11
litellm/litellm_core_utils/streaming_handler.py
View file

@ -5,7 +5,6 @@ import json
import threading
import time
import traceback
from litellm._uuid import uuid
from typing import Any, Callable, Dict, List, Optional, Union, cast
import httpx
@ -13,6 +12,7 @@ from pydantic import BaseModel
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
@ -1024,7 +1024,7 @@ class CustomStreamWrapper:
return
def chunk_creator(self, chunk: Any): # type: ignore # noqa: PLR0915
if hasattr(chunk, 'id'):
if hasattr(chunk, "id"):
self.response_id = chunk.id
model_response = self.model_response_creator()
response_obj: Dict[str, Any] = {}
@ -1365,12 +1365,13 @@ class CustomStreamWrapper:
f"model_response finish reason 3: {self.received_finish_reason}; response_obj={response_obj}"
)
## FUNCTION CALL PARSING
original_chunk = (
response_obj.get("original_chunk") if response_obj is not None else None
)
if (
response_obj is not None
and response_obj.get("original_chunk", None) is not None
original_chunk is not None
): # function / tool calling branch - only set for openai/azure compatible endpoints
# enter this branch when no content has been passed in response
original_chunk = response_obj.get("original_chunk", None)
if hasattr(original_chunk, "id"):
model_response = self.set_model_id(
original_chunk.id, model_response

6
litellm/llms/anthropic/chat/handler.py
View file

@ -51,6 +51,7 @@ from litellm.types.utils import (
ModelResponseStream,
StreamingChoices,
Usage,
_generate_id,
)
from ...base import BaseLLM
@ -490,6 +491,8 @@ class ModelResponseIterator:
self.content_blocks: List[ContentBlockDelta] = []
self.tool_index = -1
self.json_mode = json_mode
# Generate response ID once per stream to match OpenAI-compatible behavior
self.response_id = _generate_id()
# Track if we're currently streaming a response_format tool
self.is_response_format_tool: bool = False
@ -765,6 +768,7 @@ class ModelResponseIterator:
)
],
usage=usage,
id=self.response_id,
)
return returned_chunk
@ -936,4 +940,4 @@ class ModelResponseIterator:
data_json = json.loads(str_line[5:])
return self.chunk_parser(chunk=data_json)
else:
return ModelResponseStream()
return ModelResponseStream(id=self.response_id)

8
litellm/llms/anthropic/completion/transformation.py
View file

@ -55,9 +55,9 @@ class AnthropicTextConfig(BaseConfig):
to pass metadata to anthropic, it's {"user_id": "any-relevant-information"}
"""
max_tokens_to_sample: Optional[
int
] = litellm.max_tokens # anthropic requires a default
max_tokens_to_sample: Optional[int] = (
litellm.max_tokens
) # anthropic requires a default
stop_sequences: Optional[list] = None
temperature: Optional[int] = None
top_p: Optional[int] = None
@ -291,7 +291,7 @@ class AnthropicTextCompletionResponseIterator(BaseModelResponseIterator):
_chunk_text = chunk.get("completion", None)
if _chunk_text is not None and isinstance(_chunk_text, str):
text = _chunk_text
finish_reason = chunk.get("stop_reason", None)
finish_reason = chunk.get("stop_reason") or ""
if finish_reason is not None:
is_finished = True
returned_chunk = GenericStreamingChunk(

2
litellm/llms/anthropic/cost_calculation.py
View file

@ -49,7 +49,7 @@ def get_cost_for_anthropic_web_search(
## Get the cost per web search request
search_context_pricing: SearchContextCostPerQuery = (
model_info.get("search_context_cost_per_query", {}) or {}
model_info.get("search_context_cost_per_query") or SearchContextCostPerQuery()
)
cost_per_web_search_request = search_context_pricing.get(
"search_context_size_medium", 0.0

12
litellm/llms/azure/azure.py
View file

@ -182,12 +182,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
model: str,
messages: list,
model_response: ModelResponse,
api_key: str,
api_key: Optional[str],
api_base: str,
api_version: str,
api_type: str,
azure_ad_token: str,
azure_ad_token_provider: Callable,
azure_ad_token: Optional[str],
azure_ad_token_provider: Optional[Callable],
dynamic_params: bool,
print_verbose: Callable,
timeout: Union[float, httpx.Timeout],
@ -372,7 +372,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
async def acompletion(
self,
api_key: str,
api_key: Optional[str],
api_version: str,
model: str,
api_base: str,
@ -477,7 +477,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
self,
logging_obj,
api_base: str,
api_key: str,
api_key: Optional[str],
api_version: str,
dynamic_params: bool,
data: dict,
@ -555,7 +555,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
self,
logging_obj: LiteLLMLoggingObj,
api_base: str,
api_key: str,
api_key: Optional[str],
api_version: str,
dynamic_params: bool,
data: dict,

4
litellm/llms/azure/common_utils.py
View file

@ -162,8 +162,8 @@ def get_azure_ad_token_from_username_password(
def get_azure_ad_token_from_oidc(
azure_ad_token: str,
azure_client_id: Optional[str],
azure_tenant_id: Optional[str],
azure_client_id: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
scope: Optional[str] = None,
) -> str:
"""

12
litellm/llms/azure/completion/handler.py
View file

@ -30,11 +30,11 @@ class AzureTextCompletion(BaseAzureLLM):
model: str,
messages: list,
model_response: ModelResponse,
api_key: str,
api_key: Optional[str],
api_base: str,
api_version: str,
api_type: str,
azure_ad_token: str,
azure_ad_token: Optional[str],
azure_ad_token_provider: Optional[Callable],
print_verbose: Callable,
timeout,
@ -59,7 +59,7 @@ class AzureTextCompletion(BaseAzureLLM):
### CHECK IF CLOUDFLARE AI GATEWAY ###
### if so - set the model as part of the base url
if "gateway.ai.cloudflare.com" in api_base:
if api_base is not None and "gateway.ai.cloudflare.com" in api_base:
## build base url - assume api base includes resource name
client = self._init_azure_client_for_cloudflare_ai_gateway(
api_key=api_key,
@ -196,7 +196,7 @@ class AzureTextCompletion(BaseAzureLLM):
async def acompletion(
self,
api_key: str,
api_key: Optional[str],
api_version: str,
model: str,
api_base: str,
@ -263,7 +263,7 @@ class AzureTextCompletion(BaseAzureLLM):
self,
logging_obj,
api_base: str,
api_key: str,
api_key: Optional[str],
api_version: str,
data: dict,
model: str,
@ -320,7 +320,7 @@ class AzureTextCompletion(BaseAzureLLM):
self,
logging_obj,
api_base: str,
api_key: str,
api_key: Optional[str],
api_version: str,
data: dict,
model: str,

30
litellm/llms/bedrock/chat/invoke_agent/transformation.py
View file

@ -3,14 +3,15 @@ Transformation for Bedrock Invoke Agent
https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_InvokeAgent.html
"""
import base64
import json
from litellm._uuid import uuid
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
import httpx
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
@ -22,6 +23,11 @@ from litellm.types.llms.bedrock_invoke_agents import (
InvokeAgentEvent,
InvokeAgentEventHeaders,
InvokeAgentEventList,
InvokeAgentMetadata,
InvokeAgentModelInvocationInput,
InvokeAgentModelInvocationOutput,
InvokeAgentOrchestrationTrace,
InvokeAgentPreProcessingTrace,
InvokeAgentTrace,
InvokeAgentTracePayload,
InvokeAgentUsage,
@ -389,15 +395,22 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
self, trace_data: InvokeAgentTrace, usage_info: InvokeAgentUsage
) -> None:
"""Extract usage information from preprocessing trace."""
pre_processing = trace_data.get("preProcessingTrace", {})
pre_processing: Optional[InvokeAgentPreProcessingTrace] = trace_data.get(
"preProcessingTrace"
)
if not pre_processing:
return
model_output = pre_processing.get("modelInvocationOutput", {})
model_output: Optional[InvokeAgentModelInvocationOutput] = (
pre_processing.get("modelInvocationOutput")
or InvokeAgentModelInvocationOutput()
)
if not model_output:
return
metadata = model_output.get("metadata", {})
metadata: Optional[InvokeAgentMetadata] = (
model_output.get("metadata") or InvokeAgentMetadata()
)
if not metadata:
return
@ -412,11 +425,16 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
self, trace_data: InvokeAgentTrace
) -> Optional[str]:
"""Extract model information from orchestration trace."""
orchestration_trace = trace_data.get("orchestrationTrace", {})
orchestration_trace: Optional[InvokeAgentOrchestrationTrace] = trace_data.get(
"orchestrationTrace"
)
if not orchestration_trace:
return None
model_invocation = orchestration_trace.get("modelInvocationInput", {})
model_invocation: Optional[InvokeAgentModelInvocationInput] = (
orchestration_trace.get("modelInvocationInput")
or InvokeAgentModelInvocationInput()
)
if not model_invocation:
return None

32
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -7,7 +7,6 @@ import json
import time
import types
import urllib.parse
from litellm._uuid import uuid
from functools import partial
from typing import (
Any,
@ -26,6 +25,7 @@ import httpx # type: ignore
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.caching.caching import InMemoryCache
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging
@ -498,9 +498,9 @@ class BedrockLLM(BaseAWSLLM):
content=None,
)
model_response.choices[0].message = _message # type: ignore
model_response._hidden_params[
"original_response"
] = outputText # allow user to access raw anthropic tool calling response
model_response._hidden_params["original_response"] = (
outputText # allow user to access raw anthropic tool calling response
)
if (
_is_function_call is True
and stream is not None
@ -808,9 +808,9 @@ class BedrockLLM(BaseAWSLLM):
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
inference_params[k] = v
if stream is True:
inference_params[
"stream"
] = True # cohere requires stream = True in inference params
inference_params["stream"] = (
True # cohere requires stream = True in inference params
)
data = json.dumps({"prompt": prompt, **inference_params})
elif provider == "anthropic":
if model.startswith("anthropic.claude-3"):
@ -1352,9 +1352,11 @@ class AWSEventStreamDecoder:
"name": None,
"arguments": delta_obj["toolUse"]["input"],
},
"index": self.tool_calls_index
if self.tool_calls_index is not None
else index,
"index": (
self.tool_calls_index
if self.tool_calls_index is not None
else index
),
}
elif "reasoningContent" in delta_obj:
provider_specific_fields = {
@ -1384,9 +1386,11 @@ class AWSEventStreamDecoder:
"name": None,
"arguments": "{}",
},
"index": self.tool_calls_index
if self.tool_calls_index is not None
else index,
"index": (
self.tool_calls_index
if self.tool_calls_index is not None
else index
),
}
elif "stopReason" in chunk_data:
finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop"))
@ -1448,7 +1452,7 @@ class AWSEventStreamDecoder:
######### /bedrock/invoke nova mappings ###############
elif "contentBlockDelta" in chunk_data:
# when using /bedrock/invoke/nova, the chunk_data is nested under "contentBlockDelta"
_chunk_data = chunk_data.get("contentBlockDelta", None)
_chunk_data = chunk_data.get("contentBlockDelta", {})
return self.converse_chunk_parser(chunk_data=_chunk_data)
######## bedrock.mistral mappings ###############
elif "outputs" in chunk_data:

15
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -89,6 +89,7 @@ from litellm.utils import (
if TYPE_CHECKING:
from aiohttp import ClientSession
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
@ -281,7 +282,7 @@ class BaseLLMHTTPHandler:
self,
model: str,
messages: list,
api_base: str,
api_base: Optional[str],
custom_llm_provider: str,
model_response: ModelResponse,
encoding,
@ -750,7 +751,7 @@ class BaseLLMHTTPHandler:
model_response: EmbeddingResponse,
api_key: Optional[str] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
aembedding: bool = False,
aembedding: Optional[bool] = False,
headers: Optional[Dict[str, Any]] = None,
) -> EmbeddingResponse:
provider_config = ProviderConfigManager.get_provider_embedding_config(
@ -3100,7 +3101,10 @@ class BaseLLMHTTPHandler:
_is_async: bool = False,
fake_stream: bool = False,
litellm_metadata: Optional[Dict[str, Any]] = None,
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
) -> Union[
ImageResponse,
Coroutine[Any, Any, ImageResponse],
]:
"""
Handles image edit requests.
@ -3290,7 +3294,10 @@ class BaseLLMHTTPHandler:
fake_stream: bool = False,
litellm_metadata: Optional[Dict[str, Any]] = None,
api_key: Optional[str] = None,
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
) -> Union[
ImageResponse,
Coroutine[Any, Any, ImageResponse],
]:
"""
Handles image generation requests.
When _is_async=True, returns a coroutine instead of making the call directly.

62
litellm/llms/gemini/google_genai/transformation.py
View file

@ -1,6 +1,7 @@
"""
Transformation for Calling Google models in their native format.
"""
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast
import httpx
@ -25,27 +26,29 @@ else:
GenerateContentContentListUnionDict = Any
GenerateContentResponse = Any
ToolConfigDict = Any
from ..common_utils import get_api_key_from_env
class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
"""
Configuration for calling Google models in their native format.
"""
##############################
# Constants
##############################
XGOOGLE_API_KEY = "x-goog-api-key"
##############################
@property
def custom_llm_provider(self) -> Literal["gemini", "vertex_ai"]:
return "gemini"
def __init__(self):
super().__init__()
VertexLLM.__init__(self)
def get_supported_generate_content_optional_params(self, model: str) -> List[str]:
"""
Get the list of supported Google GenAI parameters for the model.
@ -58,7 +61,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
"""
return [
"http_options",
"system_instruction",
"system_instruction",
"temperature",
"top_p",
"top_k",
@ -84,10 +87,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
"speech_config",
"audio_timestamp",
"automatic_function_calling",
"thinking_config"
"thinking_config",
]
def map_generate_content_optional_params(
self,
generate_content_config_dict: GenerateContentConfigDict,
@ -103,26 +105,29 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
Returns:
Mapped parameters for the provider
"""
from litellm.types.google_genai.main import GenerateContentConfigDict
_generate_content_config_dict = GenerateContentConfigDict()
supported_google_genai_params = self.get_supported_generate_content_optional_params(model)
_generate_content_config_dict: Dict[str, Any] = {}
supported_google_genai_params = (
self.get_supported_generate_content_optional_params(model)
)
for param, value in generate_content_config_dict.items():
if param in supported_google_genai_params:
_generate_content_config_dict[param] = value
return dict(_generate_content_config_dict)
return _generate_content_config_dict
def validate_environment(
self,
self,
api_key: Optional[str],
headers: Optional[dict],
model: str,
litellm_params: Optional[Union[GenericLiteLLMParams, dict]]
litellm_params: Optional[Union[GenericLiteLLMParams, dict]],
) -> dict:
default_headers = {
"Content-Type": "application/json",
}
# Use the passed api_key first, then fall back to litellm_params and environment
gemini_api_key = api_key or self._get_google_ai_studio_api_key(dict(litellm_params or {}))
gemini_api_key = api_key or self._get_google_ai_studio_api_key(
dict(litellm_params or {})
)
if gemini_api_key is not None:
default_headers[self.XGOOGLE_API_KEY] = gemini_api_key
if headers is not None:
@ -137,14 +142,14 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
or get_api_key_from_env()
or litellm.api_key
)
def _get_common_auth_components(
self,
litellm_params: dict,
) -> Tuple[Any, Optional[str], Optional[str]]:
"""
Get common authentication components used by both sync and async methods.
Returns:
Tuple of (vertex_credentials, vertex_project, vertex_location)
"""
@ -152,7 +157,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
vertex_project = self.get_vertex_ai_project(litellm_params)
vertex_location = self.get_vertex_ai_location(litellm_params)
return vertex_credentials, vertex_project, vertex_location
def _build_final_headers_and_url(
self,
model: str,
@ -168,7 +173,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
Build final headers and API URL from auth components.
"""
gemini_api_key = self._get_google_ai_studio_api_key(litellm_params)
auth_header, api_base = self._get_token_and_url(
model=model,
gemini_api_key=gemini_api_key,
@ -201,7 +206,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
"""
Sync version of get_auth_token_and_url.
"""
vertex_credentials, vertex_project, vertex_location = self._get_common_auth_components(litellm_params)
vertex_credentials, vertex_project, vertex_location = (
self._get_common_auth_components(litellm_params)
)
_auth_header, vertex_project = self._ensure_access_token(
credentials=vertex_credentials,
@ -238,7 +245,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
Returns:
Tuple of headers and API base
"""
vertex_credentials, vertex_project, vertex_location = self._get_common_auth_components(litellm_params)
vertex_credentials, vertex_project, vertex_location = (
self._get_common_auth_components(litellm_params)
)
_auth_header, vertex_project = await self._ensure_access_token_async(
credentials=vertex_credentials,
@ -256,7 +265,6 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
api_base=api_base,
litellm_params=litellm_params,
)
def transform_generate_content_request(
self,
@ -269,6 +277,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
GenerateContentConfigDict,
GenerateContentRequestDict,
)
typed_generate_content_request = GenerateContentRequestDict(
model=model,
contents=contents,
@ -279,7 +288,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
request_dict = cast(dict, typed_generate_content_request)
return request_dict
def transform_generate_content_response(
self,
model: str,
@ -297,6 +306,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
Transformed response data
"""
from litellm.types.google_genai.main import GenerateContentResponse
try:
response = raw_response.json()
except Exception as e:
@ -305,7 +315,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
status_code=raw_response.status_code,
headers=raw_response.headers,
)
logging_obj.model_call_details["httpx_response"] = raw_response
return GenerateContentResponse(**response)
return GenerateContentResponse(**response)

7
litellm/llms/hosted_vllm/rerank/transformation.py
View file

@ -42,8 +42,11 @@ class HostedVLLMRerankConfig(BaseRerankConfig):
if api_base:
# Remove trailing slashes and ensure clean base URL
api_base = api_base.rstrip("/")
if not api_base.endswith("/v1/rerank"):
api_base = f"{api_base}/v1/rerank"
# Preserve backward compatibility
if api_base.endswith("/v1/rerank"):
api_base = api_base.replace("/v1/rerank", "/rerank")
elif not api_base.endswith("/rerank"):
api_base = f"{api_base}/rerank"
return api_base
raise ValueError("api_base must be provided for Hosted VLLM rerank")

26
litellm/llms/huggingface/embedding/transformation.py
View file

@ -40,17 +40,17 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
Reference: https://huggingface.github.io/text-generation-inference/#/Text%20Generation%20Inference/compat_generate
"""
hf_task: Optional[
hf_tasks
] = None # litellm-specific param, used to know the api spec to use when calling huggingface api
hf_task: Optional[hf_tasks] = (
None # litellm-specific param, used to know the api spec to use when calling huggingface api
)
best_of: Optional[int] = None
decoder_input_details: Optional[bool] = None
details: Optional[bool] = True # enables returning logprobs + best of
max_new_tokens: Optional[int] = None
repetition_penalty: Optional[float] = None
return_full_text: Optional[
bool
] = False # by default don't return the input as part of the output
return_full_text: Optional[bool] = (
False # by default don't return the input as part of the output
)
seed: Optional[int] = None
temperature: Optional[float] = None
top_k: Optional[int] = None
@ -120,9 +120,9 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
optional_params["top_p"] = value
if param == "n":
optional_params["best_of"] = value
optional_params[
"do_sample"
] = True # Need to sample if you want best of for hf inference endpoints
optional_params["do_sample"] = (
True # Need to sample if you want best of for hf inference endpoints
)
if param == "stream":
optional_params["stream"] = value
if param == "stop":
@ -268,7 +268,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
# check if the model has a registered custom prompt
model_prompt_details = litellm.custom_prompt_dict[model]
prompt = custom_prompt(
role_dict=model_prompt_details.get("roles", None),
role_dict=model_prompt_details.get("roles") or {},
initial_prompt_value=model_prompt_details.get(
"initial_prompt_value", ""
),
@ -363,9 +363,9 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
"content-type": "application/json",
}
if api_key is not None:
default_headers[
"Authorization"
] = f"Bearer {api_key}" # Huggingface Inference Endpoint default is to accept bearer tokens
default_headers["Authorization"] = (
f"Bearer {api_key}" # Huggingface Inference Endpoint default is to accept bearer tokens
)
headers = {**headers, **default_headers}
return headers

12
litellm/llms/openai/completion/transformation.py
View file

@ -1,5 +1,5 @@
"""
Support for gpt model family
Support for gpt model family
"""
from typing import List, Optional, Union
@ -87,7 +87,7 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig):
## RESPONSE OBJECT
if response_object is None or model_response_object is None:
raise ValueError("Error in response object format")
choice_list = []
choice_list: List[Choices] = []
for idx, choice in enumerate(response_object["choices"]):
message = Message(
content=choice["text"],
@ -100,7 +100,7 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig):
logprobs=choice.get("logprobs", None),
)
choice_list.append(choice)
model_response_object.choices = choice_list
model_response_object.choices = choice_list # type: ignore
if "usage" in response_object:
setattr(model_response_object, "usage", response_object["usage"])
@ -111,9 +111,9 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig):
if "model" in response_object:
model_response_object.model = response_object["model"]
model_response_object._hidden_params[
"original_response"
] = response_object # track original response, if users make a litellm.text_completion() request, we can return the original response
model_response_object._hidden_params["original_response"] = (
response_object # track original response, if users make a litellm.text_completion() request, we can return the original response
)
return model_response_object
except Exception as e:
raise e

14
litellm/llms/openai/image_edit/transformation.py
View file

@ -91,21 +91,22 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
# Handle image parameter
if _image_list is not None:
image_list = [_image_list] if not isinstance(_image_list, list) else _image_list
image_list = (
[_image_list] if not isinstance(_image_list, list) else _image_list
)
for _image in image_list:
if _image is not None:
image_content_type: str = ImageEditRequestUtils.get_image_content_type(
_image
image_content_type: str = (
ImageEditRequestUtils.get_image_content_type(_image)
)
if isinstance(_image, BufferedReader):
files_list.append(
("image", (_image.name, _image, image_content_type))
("image[]", (_image.name, _image, image_content_type))
)
else:
files_list.append(
("image", ("image.png", _image, image_content_type))
("image[]", ("image.png", _image, image_content_type))
)
# Handle mask parameter if provided
if _mask is not None:
# Handle case where mask can be a list (extract first mask)
@ -120,6 +121,7 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
files_list.append(("mask", (_mask.name, _mask, mask_content_type)))
else:
files_list.append(("mask", ("mask.png", _mask, mask_content_type)))
return data_without_files, files_list
def transform_image_edit_response(

16
litellm/llms/vertex_ai/fine_tuning/handler.py
View file

@ -64,9 +64,9 @@ class VertexFineTuningAPI(VertexLLM):
)
if create_fine_tuning_job_data.validation_file:
supervised_tuning_spec[
"validation_dataset"
] = create_fine_tuning_job_data.validation_file
supervised_tuning_spec["validation_dataset"] = (
create_fine_tuning_job_data.validation_file
)
_vertex_hyperparameters = (
self._transform_openai_hyperparameters_to_vertex_hyperparameters(
@ -140,7 +140,9 @@ class VertexFineTuningAPI(VertexLLM):
fine_tuned_model=response.get("tunedModelDisplayName", ""),
finished_at=None,
hyperparameters=self._translate_vertex_response_hyperparameters(
vertex_hyper_parameters=_supervisedTuningSpec.get("hyperParameters", {})
vertex_hyper_parameters=_supervisedTuningSpec.get(
"hyperParameters", FineTuneHyperparameters()
)
or {}
),
model=response.get("baseModel", "") or "",
@ -343,9 +345,9 @@ class VertexFineTuningAPI(VertexLLM):
elif "cachedContents" in request_route:
_model = request_data.get("model")
if _model is not None and "/publishers/google/models/" not in _model:
request_data[
"model"
] = f"projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{_model}"
request_data["model"] = (
f"projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{_model}"
)
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
else:

2
litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py
View file

@ -43,7 +43,7 @@ class GoogleBatchEmbeddings(VertexLLM):
vertex_project=None,
vertex_location=None,
vertex_credentials=None,
aembedding=False,
aembedding: Optional[bool] = False,
timeout=300,
client=None,
) -> EmbeddingResponse:

14
litellm/llms/vertex_ai/google_genai/transformation.py
View file

@ -1,7 +1,8 @@
"""
Transformation for Calling Google models in their native format.
"""
from typing import Dict, Literal, Optional, Union
from typing import Any, Dict, Literal, Optional, Union
from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig
from litellm.types.router import GenericLiteLLMParams
@ -58,22 +59,21 @@ class VertexAIGoogleGenAIConfig(GoogleGenAIConfig):
Returns:
Mapped parameters for the provider
"""
from litellm.types.google_genai.main import GenerateContentConfigDict
_generate_content_config_dict = GenerateContentConfigDict()
_generate_content_config_dict: Dict = {}
for param, value in generate_content_config_dict.items():
camel_case_key = self._camel_to_snake(param)
_generate_content_config_dict[camel_case_key] = value
return dict(_generate_content_config_dict)
return _generate_content_config_dict
def transform_generate_content_request(
self,
model: str,
contents: any,
tools: Optional[any],
contents: Any,
tools: Optional[Any],
generate_content_config_dict: Dict,
system_instruction: Optional[any] = None,
system_instruction: Optional[Any] = None,
) -> dict:
"""
Transform the generate content request for Vertex AI.

2
litellm/llms/vertex_ai/multimodal_embeddings/embedding_handler.py
View file

@ -46,7 +46,7 @@ class VertexMultimodalEmbedding(VertexLLM):
vertex_project=None,
vertex_location=None,
vertex_credentials=None,
aembedding=False,
aembedding: Optional[bool] = False,
timeout=300,
client=None,
) -> EmbeddingResponse:

14
litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py
View file

@ -36,7 +36,7 @@ class VertexEmbedding(VertexBase):
timeout: Optional[Union[float, httpx.Timeout]],
api_key: Optional[str] = None,
encoding=None,
aembedding=False,
aembedding: Optional[bool] = False,
api_base: Optional[str] = None,
client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None,
vertex_project: Optional[str] = None,
@ -86,8 +86,10 @@ class VertexEmbedding(VertexBase):
mode="embedding",
)
headers = self.set_headers(auth_header=auth_header, extra_headers=extra_headers)
vertex_request: VertexEmbeddingRequest = litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request(
input=input, optional_params=optional_params, model=model
vertex_request: VertexEmbeddingRequest = (
litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request(
input=input, optional_params=optional_params, model=model
)
)
_client_params = {}
@ -176,8 +178,10 @@ class VertexEmbedding(VertexBase):
mode="embedding",
)
headers = self.set_headers(auth_header=auth_header, extra_headers=extra_headers)
vertex_request: VertexEmbeddingRequest = litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request(
input=input, optional_params=optional_params, model=model
vertex_request: VertexEmbeddingRequest = (
litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request(
input=input, optional_params=optional_params, model=model
)
)
_async_client_params = {}

4
litellm/llms/watsonx/chat/handler.py
View file

@ -21,7 +21,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler):
*,
model: str,
messages: list,
api_base: str,
api_base: Optional[str],
custom_llm_provider: str,
custom_prompt_dict: dict,
model_response: ModelResponse,
@ -70,7 +70,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler):
)
return super().completion(
model=watsonx_auth_payload.get("model_id", None),
model=watsonx_auth_payload.get("model_id") or "",
messages=messages,
api_base=api_base,
custom_llm_provider=custom_llm_provider,

60
litellm/main.py
View file

@ -17,12 +17,12 @@ import random
import sys
import time
import traceback
from litellm._uuid import uuid
from concurrent import futures
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from copy import deepcopy
from functools import partial
from typing import (
TYPE_CHECKING,
Any,
Callable,
Coroutine,
@ -36,9 +36,10 @@ from typing import (
Union,
cast,
get_args,
TYPE_CHECKING,
)
from litellm._uuid import uuid
if TYPE_CHECKING:
from aiohttp import ClientSession
@ -721,12 +722,15 @@ async def _sleep_for_timeout_async(timeout: Union[float, str, httpx.Timeout]):
await asyncio.sleep(timeout.connect)
MOCK_RESPONSE_TYPE = Union[str, Exception, dict]
def mock_completion(
model: str,
messages: List,
stream: Optional[bool] = False,
n: Optional[int] = None,
mock_response: Union[str, Exception, dict] = "This is a mock request",
mock_response: Optional[MOCK_RESPONSE_TYPE] = "This is a mock request",
mock_tool_calls: Optional[List] = None,
mock_timeout: Optional[bool] = False,
logging=None,
@ -1007,7 +1011,7 @@ def completion( # type: ignore # noqa: PLR0915
######### unpacking kwargs #####################
args = locals()
api_base = kwargs.get("api_base", None)
mock_response = kwargs.get("mock_response", None)
mock_response: Optional[MOCK_RESPONSE_TYPE] = kwargs.get("mock_response", None)
mock_tool_calls = kwargs.get("mock_tool_calls", None)
mock_timeout = cast(Optional[bool], kwargs.get("mock_timeout", None))
force_timeout = kwargs.get("force_timeout", 600) ## deprecated
@ -1114,7 +1118,7 @@ def completion( # type: ignore # noqa: PLR0915
api_base = base_url
if num_retries is not None:
max_retries = num_retries
logging = litellm_logging_obj
logging: Logging = cast(Logging, litellm_logging_obj)
fallbacks = fallbacks or litellm.model_fallbacks
if fallbacks is not None:
return completion_with_fallbacks(**args)
@ -1427,7 +1431,7 @@ def completion( # type: ignore # noqa: PLR0915
api_version = (
api_version
or litellm.api_version
or get_secret("AZURE_API_VERSION")
or get_secret_str("AZURE_API_VERSION")
or litellm.AZURE_DEFAULT_API_VERSION
)
@ -1435,13 +1439,13 @@ def completion( # type: ignore # noqa: PLR0915
api_key
or litellm.api_key
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
azure_ad_token = optional_params.get("extra_body", {}).pop(
"azure_ad_token", None
) or get_secret("AZURE_AD_TOKEN")
) or get_secret_str("AZURE_AD_TOKEN")
azure_ad_token_provider = litellm_params.get(
"azure_ad_token_provider", None
@ -1529,25 +1533,32 @@ def completion( # type: ignore # noqa: PLR0915
)
elif custom_llm_provider == "azure_text":
# azure configs
api_type = get_secret("AZURE_API_TYPE") or "azure"
api_type = get_secret_str("AZURE_API_TYPE") or "azure"
api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
if api_base is None:
raise ValueError(
"api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable."
)
api_version = (
api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
)
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
azure_ad_token = optional_params.get("extra_body", {}).pop(
"azure_ad_token", None
) or get_secret("AZURE_AD_TOKEN")
) or get_secret_str("AZURE_AD_TOKEN")
azure_ad_token_provider = litellm_params.get(
"azure_ad_token_provider", None
@ -1573,7 +1584,7 @@ def completion( # type: ignore # noqa: PLR0915
headers=headers,
api_key=api_key,
api_base=api_base,
api_version=api_version,
api_version=cast(str, api_version),
api_type=api_type,
azure_ad_token=azure_ad_token,
azure_ad_token_provider=azure_ad_token_provider,
@ -2545,15 +2556,10 @@ def completion( # type: ignore # noqa: PLR0915
)
elif custom_llm_provider == "compactifai":
api_key = (
api_key
or get_secret_str("COMPACTIFAI_API_KEY")
or litellm.api_key
api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key
)
api_base = (
api_base
or "https://api.compactif.ai/v1"
)
api_base = api_base or "https://api.compactif.ai/v1"
## COMPLETION CALL
response = base_llm_http_handler.completion(
@ -2860,7 +2866,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
acompletion=acompletion,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
custom_llm_provider=custom_llm_provider, # type: ignore
client=client,
api_base=api_base,
extra_headers=extra_headers,
@ -2929,7 +2935,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
acompletion=acompletion,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
custom_llm_provider=custom_llm_provider, # type: ignore
client=client,
api_base=api_base,
extra_headers=extra_headers,
@ -3935,7 +3941,7 @@ def embedding( # noqa: PLR0915
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
mock_response: Optional[List[float]] = kwargs.get("mock_response", None) # type: ignore
azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None)
aembedding = kwargs.get("aembedding", None)
aembedding: Optional[bool] = kwargs.get("aembedding", None)
extra_headers = kwargs.get("extra_headers", None)
headers = kwargs.get("headers", None)
### CUSTOM MODEL COST ###
@ -5615,7 +5621,7 @@ def speech( # noqa: PLR0915
if max_retries is None:
max_retries = litellm.num_retries or openai.DEFAULT_MAX_RETRIES
litellm_params_dict = get_litellm_params(**kwargs)
logging_obj = kwargs.get("litellm_logging_obj", None)
logging_obj: Logging = cast(Logging, kwargs.get("litellm_logging_obj"))
logging_obj.update_environment_variables(
model=model,
user=user,

26
litellm/model_prices_and_context_window_backup.json
View file

@ -12735,7 +12735,7 @@
"input_cost_per_token_flex": 6.25e-07,
"input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -12773,7 +12773,7 @@
"input_cost_per_token_flex": 6.25e-07,
"input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -12808,7 +12808,7 @@
"cache_read_input_token_cost": 1.25e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -12840,7 +12840,7 @@
"cache_read_input_token_cost": 1.25e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -12906,7 +12906,7 @@
"input_cost_per_token_flex": 1.25e-07,
"input_cost_per_token_priority": 4.5e-07,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -12944,7 +12944,7 @@
"input_cost_per_token_flex": 1.25e-07,
"input_cost_per_token_priority": 4.5e-07,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -12981,7 +12981,7 @@
"input_cost_per_token_flex": 2.5e-08,
"input_cost_per_token_priority": 2.5e-06,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -13016,7 +13016,7 @@
"input_cost_per_token": 5e-08,
"input_cost_per_token_flex": 2.5e-08,
"litellm_provider": "openai",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -17210,7 +17210,7 @@
"cache_read_input_token_cost": 1.25e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -17229,7 +17229,7 @@
"cache_read_input_token_cost": 1.25e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -17248,7 +17248,7 @@
"cache_read_input_token_cost": 1.25e-07,
"input_cost_per_token": 1.25e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -17267,7 +17267,7 @@
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_token": 2.5e-07,
"litellm_provider": "openrouter",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
@ -17286,7 +17286,7 @@
"cache_read_input_token_cost": 5e-09,
"input_cost_per_token": 5e-08,
"litellm_provider": "openrouter",
"max_input_tokens": 400000,
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",

7
litellm/mypy.ini
View file

@ -5,10 +5,15 @@ mypy_path = litellm/stubs
namespace_packages = True
disable_error_code =
valid-type,
annotation-unchecked
annotation-unchecked,
import-untyped
[mypy-google.*]
ignore_missing_imports = True
[mypy-cryptography.hazmat.bindings._rust.x509]
ignore_errors = True
[mypy-fastuuid.*]
ignore_missing_imports = True
ignore_errors = True

16
litellm/proxy/_experimental/mcp_server/auth/litellm_auth_handler.py
View file

@ -1,4 +1,4 @@
from typing import List, Optional, Dict
from typing import Dict, List, Optional
from mcp.server.auth.middleware.bearer_auth import AuthenticatedUser
@ -8,17 +8,27 @@ from litellm.proxy._types import UserAPIKeyAuth
class MCPAuthenticatedUser(AuthenticatedUser):
"""
Wrapper class to make LiteLLM's authentication and configuration compatible with MCP's AuthenticatedUser.
This class handles:
1. User API key authentication information
2. MCP authentication header (deprecated)
3. MCP server configuration (can include access groups)
4. Server-specific authentication headers
5. OAuth2 headers
"""
def __init__(self, user_api_key_auth: UserAPIKeyAuth, mcp_auth_header: Optional[str] = None, mcp_servers: Optional[List[str]] = None, mcp_server_auth_headers: Optional[Dict[str, str]] = None, mcp_protocol_version: Optional[str] = None):
def __init__(
self,
user_api_key_auth: UserAPIKeyAuth,
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
mcp_protocol_version: Optional[str] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
):
self.user_api_key_auth = user_api_key_auth
self.mcp_auth_header = mcp_auth_header
self.mcp_servers = mcp_servers
self.mcp_server_auth_headers = mcp_server_auth_headers or {}
self.mcp_protocol_version = mcp_protocol_version
self.oauth2_headers = oauth2_headers

49
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -1,4 +1,4 @@
from typing import List, Optional, Tuple, Dict, Set
from typing import Dict, List, Optional, Set, Tuple
from starlette.datastructures import Headers
from starlette.requests import Request
@ -36,7 +36,11 @@ class MCPRequestHandler:
async def process_mcp_request(
scope: Scope,
) -> Tuple[
UserAPIKeyAuth, Optional[str], Optional[List[str]], Optional[Dict[str, str]]
UserAPIKeyAuth,
Optional[str],
Optional[List[str]],
Optional[Dict[str, str]],
Optional[Dict[str, str]],
]:
"""
Process and validate MCP request headers from the ASGI scope.
@ -44,6 +48,7 @@ class MCPRequestHandler:
1. Extracting and validating authentication headers
2. Processing MCP server configuration
3. Handling MCP-specific headers
4. Handling oauth2 headers
Args:
scope: ASGI scope containing request information
@ -70,6 +75,9 @@ class MCPRequestHandler:
MCPRequestHandler._get_mcp_server_auth_headers_from_headers(headers)
)
# Get the oauth2 headers
oauth2_headers = MCPRequestHandler._get_oauth2_headers_from_headers(headers)
# Parse MCP servers from header
mcp_servers_header = headers.get(
MCPRequestHandler.LITELLM_MCP_SERVERS_HEADER_NAME
@ -96,14 +104,18 @@ class MCPRequestHandler:
return b"{}"
request.body = mock_body # type: ignore
validated_user_api_key_auth = await user_api_key_auth(
api_key=litellm_api_key, request=request
)
if ".well-known" in str(request.url): # public routes
validated_user_api_key_auth = UserAPIKeyAuth()
else:
validated_user_api_key_auth = await user_api_key_auth(
api_key=litellm_api_key, request=request
)
return (
validated_user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
)
@staticmethod
@ -174,6 +186,17 @@ class MCPRequestHandler:
return server_auth_headers
@staticmethod
def _get_oauth2_headers_from_headers(headers: Headers) -> Dict[str, str]:
"""
Get the oauth2 headers from the request headers.
"""
oauth2_headers = {}
for header_name, header_value in headers.items():
if header_name.lower().startswith("authorization"):
oauth2_headers["Authorization"] = header_value
return oauth2_headers
@staticmethod
def _get_mcp_client_side_auth_header_name() -> str:
"""
@ -359,10 +382,10 @@ class MCPRequestHandler:
return []
try:
team_obj: Optional[
LiteLLM_TeamTable
] = await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
team_obj: Optional[LiteLLM_TeamTable] = (
await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
)
if team_obj is None:
verbose_logger.debug("team_obj is None")
@ -535,10 +558,10 @@ class MCPRequestHandler:
verbose_logger.debug("prisma_client is None")
return []
team_obj: Optional[
LiteLLM_TeamTable
] = await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
team_obj: Optional[LiteLLM_TeamTable] = (
await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": user_api_key_auth.team_id},
)
)
if team_obj is None:
verbose_logger.debug("team_obj is None")

35
litellm/proxy/_experimental/mcp_server/cost_calculator.py
View file

@ -1,6 +1,7 @@
"""
Cost calculator for MCP tools.
"""
from typing import TYPE_CHECKING, Any, Optional, cast
from litellm.types.mcp import MCPServerCostInfo
@ -13,11 +14,12 @@ if TYPE_CHECKING:
else:
LitellmLoggingObject = Any
class MCPCostCalculator:
@staticmethod
def calculate_mcp_tool_call_cost(
litellm_logging_obj: Optional[LitellmLoggingObject],
) -> float:
) -> float:
"""
Calculate the cost of an MCP tool call.
@ -25,28 +27,43 @@ class MCPCostCalculator:
"""
if litellm_logging_obj is None:
return 0.0
#########################################################
# Get the response cost from logging object model_call_details
# This is set when a user modifies the response in a post_mcp_tool_call_hook
#########################################################
response_cost = litellm_logging_obj.model_call_details.get("response_cost", None)
response_cost = litellm_logging_obj.model_call_details.get(
"response_cost", None
)
if response_cost is not None:
return response_cost
#########################################################
# Unpack the mcp_tool_call_metadata
#########################################################
mcp_tool_call_metadata: StandardLoggingMCPToolCall = cast(StandardLoggingMCPToolCall, litellm_logging_obj.model_call_details.get("mcp_tool_call_metadata", {})) or {}
mcp_server_cost_info: MCPServerCostInfo = mcp_tool_call_metadata.get("mcp_server_cost_info", {}) or {}
mcp_tool_call_metadata: StandardLoggingMCPToolCall = (
cast(
StandardLoggingMCPToolCall,
litellm_logging_obj.model_call_details.get(
"mcp_tool_call_metadata", {}
),
)
or {}
)
mcp_server_cost_info: MCPServerCostInfo = (
mcp_tool_call_metadata.get("mcp_server_cost_info") or MCPServerCostInfo()
)
#########################################################
# User defined cost per query
#########################################################
default_cost_per_query = mcp_server_cost_info.get("default_cost_per_query", None)
tool_name_to_cost_per_query: dict = mcp_server_cost_info.get("tool_name_to_cost_per_query", {}) or {}
default_cost_per_query = mcp_server_cost_info.get(
"default_cost_per_query", None
)
tool_name_to_cost_per_query: dict = (
mcp_server_cost_info.get("tool_name_to_cost_per_query", {}) or {}
)
tool_name = mcp_tool_call_metadata.get("name", "")
#########################################################
# 1. If tool_name is in tool_name_to_cost_per_query, use the cost per query
# 2. If tool_name is not in tool_name_to_cost_per_query, use the default cost per query

252
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py Normal file
View file

@ -0,0 +1,252 @@
import json
from typing import Optional, Tuple
from urllib.parse import urlencode, urlparse, urlunparse
from fastapi import APIRouter, Form, HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
)
from litellm.proxy.common_utils.encrypt_decrypt_utils import (
decrypt_value_helper,
encrypt_value_helper,
)
router = APIRouter(
tags=["mcp"],
)
def encode_state_with_base_url(base_url: str, original_state: str) -> str:
"""
Encode the base_url and original state using encryption.
Args:
base_url: The base URL to encode
original_state: The original state parameter
Returns:
An encrypted string that encodes both values
"""
state_data = {"base_url": base_url, "original_state": original_state}
state_json = json.dumps(state_data, sort_keys=True)
encrypted_state = encrypt_value_helper(state_json)
return encrypted_state
def decode_state_hash(encrypted_state: str) -> Tuple[str, str]:
"""
Decode an encrypted state to retrieve the base_url and original state.
Args:
encrypted_state: The encrypted string to decode
Returns:
A tuple of (base_url, original_state)
Raises:
Exception: If decryption fails or data is malformed
"""
decrypted_json = decrypt_value_helper(encrypted_state, "oauth_state")
if decrypted_json is None:
raise ValueError("Failed to decrypt state parameter")
state_data = json.loads(decrypted_json)
return state_data["base_url"], state_data["original_state"]
@router.get("/{mcp_server_name}/authorize")
@router.get("/authorize")
async def authorize(
request: Request,
client_id: str,
redirect_uri: str,
state: str = "",
mcp_server_name: Optional[str] = None,
):
# Redirect to real GitHub OAuth
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(client_id)
if mcp_server is None:
raise HTTPException(status_code=404, detail="MCP server not found")
if mcp_server.auth_type != "oauth2":
raise HTTPException(status_code=400, detail="MCP server is not OAuth2")
if mcp_server.client_id is None:
raise HTTPException(status_code=400, detail="MCP server client id is not set")
if mcp_server.authorization_url is None:
raise HTTPException(
status_code=400, detail="MCP server authorization url is not set"
)
if mcp_server.scopes is None:
raise HTTPException(status_code=400, detail="MCP server scopes is not set")
# Parse it to remove any existing query
parsed = urlparse(redirect_uri)
base_url = urlunparse(parsed._replace(query=""))
request_base_url = str(request.base_url).rstrip("/")
# Encode the base_url and original state in a unique hash
encoded_state = encode_state_with_base_url(base_url, state)
params = {
"client_id": mcp_server.client_id,
"redirect_uri": f"{request_base_url}/callback",
"scope": " ".join(mcp_server.scopes),
"state": encoded_state,
}
return RedirectResponse(f"{mcp_server.authorization_url}?{urlencode(params)}")
@router.post("/token")
async def token_endpoint(
request: Request,
grant_type: str = Form(...),
code: str = Form(None),
redirect_uri: str = Form(None),
client_id: str = Form(...),
client_secret: str = Form(...),
):
"""
Accept the authorization code from Claude and exchange it for GitHub token.
Forward the GitHub token back to Claude in standard OAuth format.
1. Call the token endpoint
2. Store the user's PAT in the db - and generate a LiteLLM virtual key
2. Return the token
3. Return a virtual key in this response
"""
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(client_id)
if mcp_server is None:
raise HTTPException(status_code=404, detail="MCP server not found")
if grant_type != "authorization_code":
raise HTTPException(status_code=400, detail="Unsupported grant_type")
if mcp_server.token_url is None:
raise HTTPException(status_code=400, detail="MCP server token url is not set")
proxy_base_url = str(request.base_url).rstrip("/")
# Exchange code for real GitHub token
async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check)
response = await async_client.post(
mcp_server.token_url,
headers={"Accept": "application/json"},
data={
"client_id": mcp_server.client_id,
"client_secret": mcp_server.client_secret,
"code": code,
"redirect_uri": f"{proxy_base_url}/callback",
},
)
response.raise_for_status()
github_token = response.json()["access_token"]
# Return to Claude in expected OAuth 2 format
### return a virtual key in this response
return JSONResponse(
{"access_token": github_token, "token_type": "Bearer", "expires_in": 3600}
)
@router.get("/callback")
async def callback(code: str, state: str):
try:
# Decode the state hash to get base_url and original state
base_url, original_state = decode_state_hash(state)
# Exchange code for token with GitHub
params = {"code": code, "state": original_state}
# Forward token to Claude ephemeral endpoint
complete_returned_url = f"{base_url}?{urlencode(params)}"
return RedirectResponse(url=complete_returned_url, status_code=302)
except Exception:
# fallback if state hash not found
return HTMLResponse(
"<html><body>Authentication incomplete. You can close this window.</body></html>"
)
# ------------------------------
# Optional .well-known endpoints for MCP + OAuth discovery
# ------------------------------
@router.get("/.well-known/oauth-protected-resource/{mcp_server_name}/mcp")
@router.get("/.well-known/oauth-protected-resource")
async def oauth_protected_resource_mcp(
request: Request, mcp_server_name: Optional[str] = None
):
request_base_url = str(request.base_url).rstrip("/")
return {
"authorization_servers": [
(
f"{request_base_url}/{mcp_server_name}"
if mcp_server_name
else f"{request_base_url}"
)
],
"resource": (
f"{request_base_url}/{mcp_server_name}/mcp"
if mcp_server_name
else f"{request_base_url}/mcp"
), # this is what Claude will call
}
@router.get("/.well-known/oauth-authorization-server/{mcp_server_name}")
@router.get("/.well-known/oauth-authorization-server")
async def oauth_authorization_server_mcp(
request: Request, mcp_server_name: Optional[str] = None
):
request_base_url = str(request.base_url).rstrip("/")
return {
"issuer": request_base_url, # point to your proxy
"authorization_endpoint": f"{request_base_url}/authorize",
"token_endpoint": f"{request_base_url}/token",
"response_types_supported": ["code"],
"grant_types_supported": ["authorization_code"],
"code_challenge_methods_supported": ["S256"],
"token_endpoint_auth_methods_supported": ["client_secret_post"],
# Claude expects a registration endpoint, even if we just fake it
"registration_endpoint": f"{request_base_url}/{mcp_server_name}/register",
}
# Alias for standard OpenID discovery
@router.get("/.well-known/openid-configuration")
async def openid_configuration(request: Request):
return await oauth_authorization_server_mcp(request)
@router.get("/.well-known/oauth-authorization-server/{mcp_server_name}/mcp")
@router.get("/.well-known/oauth-authorization-server")
async def oauth_authorization_server_root(
request: Request, mcp_server_name: Optional[str] = None
):
return await oauth_authorization_server_mcp(request, mcp_server_name)
@router.post("/{mcp_server_name}/register")
@router.post("/register")
async def register_client(request: Request, mcp_server_name: Optional[str] = None):
request_base_url = str(request.base_url).rstrip("/")
# return fixed GitHub client credentials
return {
"client_id": mcp_server_name or "dummy_client",
"client_secret": "dummy",
"redirect_uris": [f"{request_base_url}/mcp/callback"],
}

172
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -39,7 +39,7 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.utils import ProxyLogging
from litellm.types.mcp import MCPStdioConfig
from litellm.types.mcp import MCPAuth, MCPStdioConfig
from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer
@ -199,6 +199,12 @@ class MCPServerManager:
command=server_config.get("command", None) or "",
args=server_config.get("args", None) or [],
env=server_config.get("env", None) or {},
# oauth specific fields
client_id=server_config.get("client_id", None),
client_secret=server_config.get("client_secret", None),
scopes=server_config.get("scopes", None),
authorization_url=server_config.get("authorization_url", None),
token_url=server_config.get("token_url", None),
# TODO: utility fn the default values
transport=server_config.get("transport", MCPTransport.http),
auth_type=server_config.get("auth_type", None),
@ -376,6 +382,7 @@ class MCPServerManager:
self,
server: MCPServer,
mcp_auth_header: Optional[str] = None,
extra_headers: Optional[Dict[str, str]] = None,
) -> MCPClient:
"""
Create an MCPClient instance for the given server.
@ -405,6 +412,7 @@ class MCPServerManager:
auth_value=mcp_auth_header or server.authentication_token,
timeout=60.0,
stdio_config=stdio_config,
extra_headers=extra_headers,
)
else:
# For HTTP/SSE transports
@ -415,12 +423,14 @@ class MCPServerManager:
auth_type=server.auth_type,
auth_value=mcp_auth_header or server.authentication_token,
timeout=60.0,
extra_headers=extra_headers,
)
async def _get_tools_from_server(
self,
server: MCPServer,
mcp_auth_header: Optional[str] = None,
extra_headers: Optional[Dict[str, str]] = None,
) -> List[MCPTool]:
"""
Helper method to get tools from a single MCP server with prefixed names.
@ -441,6 +451,7 @@ class MCPServerManager:
client = self._create_mcp_client(
server=server,
mcp_auth_header=mcp_auth_header,
extra_headers=extra_headers,
)
tools = await self._fetch_tools_with_timeout(client, server.name)
@ -550,6 +561,77 @@ class MCPServerManager:
)
return prefixed_tools
async def pre_call_tool_check(
self,
name: str,
arguments: Dict[str, Any],
server_name_from_prefix: str,
user_api_key_auth: Optional[UserAPIKeyAuth],
proxy_logging_obj: ProxyLogging,
):
pre_hook_kwargs = {
"name": name,
"arguments": arguments,
"server_name": server_name_from_prefix,
"user_api_key_auth": user_api_key_auth,
"user_api_key_user_id": (
getattr(user_api_key_auth, "user_id", None)
if user_api_key_auth
else None
),
"user_api_key_team_id": (
getattr(user_api_key_auth, "team_id", None)
if user_api_key_auth
else None
),
"user_api_key_end_user_id": (
getattr(user_api_key_auth, "end_user_id", None)
if user_api_key_auth
else None
),
"user_api_key_hash": (
getattr(user_api_key_auth, "api_key_hash", None)
if user_api_key_auth
else None
),
}
# Create MCP request object for processing
mcp_request_obj = proxy_logging_obj._create_mcp_request_object_from_kwargs(
pre_hook_kwargs
)
# Convert to LLM format for existing guardrail compatibility
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(
mcp_request_obj, pre_hook_kwargs
)
try:
# Use standard pre_call_hook with call_type="mcp_call"
modified_data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_auth, # type: ignore
data=synthetic_llm_data,
call_type="mcp_call", # type: ignore
)
if modified_data:
# Convert response back to MCP format and apply modifications
modified_kwargs = (
proxy_logging_obj._convert_mcp_hook_response_to_kwargs(
modified_data, pre_hook_kwargs
)
)
if modified_kwargs.get("arguments") != arguments:
arguments = modified_kwargs["arguments"]
except (
BlockedPiiEntityError,
GuardrailRaisedException,
HTTPException,
) as e:
# Re-raise guardrail exceptions to properly fail the MCP call
verbose_logger.error(f"Guardrail blocked MCP tool call pre call: {str(e)}")
raise e
async def call_tool(
self,
name: str,
@ -558,6 +640,7 @@ class MCPServerManager:
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
proxy_logging_obj: Optional[ProxyLogging] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
) -> CallToolResult:
"""
Call a tool with the given name and arguments (handles prefixed tool names)
@ -602,65 +685,14 @@ class MCPServerManager:
# Using standard pre_call_hook with call_type="mcp_call"
#########################################################
if proxy_logging_obj:
pre_hook_kwargs = {
"name": name,
"arguments": arguments,
"server_name": server_name_from_prefix,
"user_api_key_auth": user_api_key_auth,
"user_api_key_user_id": getattr(user_api_key_auth, "user_id", None)
if user_api_key_auth
else None,
"user_api_key_team_id": getattr(user_api_key_auth, "team_id", None)
if user_api_key_auth
else None,
"user_api_key_end_user_id": getattr(
user_api_key_auth, "end_user_id", None
)
if user_api_key_auth
else None,
"user_api_key_hash": getattr(user_api_key_auth, "api_key_hash", None)
if user_api_key_auth
else None,
}
# Create MCP request object for processing
mcp_request_obj = proxy_logging_obj._create_mcp_request_object_from_kwargs(
pre_hook_kwargs
await self.pre_call_tool_check(
name=original_tool_name,
arguments=arguments,
server_name_from_prefix=server_name_from_prefix,
user_api_key_auth=user_api_key_auth,
proxy_logging_obj=proxy_logging_obj,
)
# Convert to LLM format for existing guardrail compatibility
synthetic_llm_data = proxy_logging_obj._convert_mcp_to_llm_format(
mcp_request_obj, pre_hook_kwargs
)
try:
# Use standard pre_call_hook with call_type="mcp_call"
modified_data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_auth, # type: ignore
data=synthetic_llm_data,
call_type="mcp_call", # type: ignore
)
if modified_data:
# Convert response back to MCP format and apply modifications
modified_kwargs = (
proxy_logging_obj._convert_mcp_hook_response_to_kwargs(
modified_data, pre_hook_kwargs
)
)
if modified_kwargs.get("arguments") != arguments:
arguments = modified_kwargs["arguments"]
except (
BlockedPiiEntityError,
GuardrailRaisedException,
HTTPException,
) as e:
# Re-raise guardrail exceptions to properly fail the MCP call
verbose_logger.error(
f"Guardrail blocked MCP tool call pre call: {str(e)}"
)
raise e
# Get server-specific auth header if available
server_auth_header = None
if mcp_server_auth_headers and mcp_server.alias:
@ -672,9 +704,15 @@ class MCPServerManager:
if server_auth_header is None:
server_auth_header = mcp_auth_header
# oauth2 headers
extra_headers: Optional[Dict[str, str]] = None
if mcp_server.auth_type == MCPAuth.oauth2:
extra_headers = oauth2_headers
client = self._create_mcp_client(
server=mcp_server,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
)
async with client:
@ -834,6 +872,16 @@ class MCPServerManager:
return server
return None
def get_mcp_server_by_name(self, server_name: str) -> Optional[MCPServer]:
"""
Get the MCP Server from the server name
"""
registry = self.get_registry()
for server in registry.values():
if server.server_name == server_name:
return server
return None
def _generate_stable_server_id(
self,
server_name: str,
@ -1023,9 +1071,11 @@ class MCPServerManager:
auth_type=_server_config.auth_type,
created_at=datetime.datetime.now(),
updated_at=datetime.datetime.now(),
description=_server_config.mcp_info.get("description")
if _server_config.mcp_info
else None,
description=(
_server_config.mcp_info.get("description")
if _server_config.mcp_info
else None
),
mcp_info=_server_config.mcp_info,
mcp_access_groups=_server_config.access_groups or [],
# Stdio-specific fields

6
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -177,9 +177,9 @@ if MCP_AVAILABLE:
return {
"tools": list_tools_result,
"error": "partial_failure" if error_message else None,
"message": error_message
if error_message
else "Successfully retrieved tools",
"message": (
error_message if error_message else "Successfully retrieved tools"
),
}
except Exception as e:

90
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -22,6 +22,7 @@ from litellm.proxy._experimental.mcp_server.utils import (
LITELLM_MCP_SERVER_VERSION,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.mcp import MCPAuth
from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer
from litellm.types.utils import StandardLoggingMCPToolCall
from litellm.utils import client
@ -178,6 +179,7 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
) = get_auth_context()
verbose_logger.debug(
f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}"
@ -195,6 +197,7 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
)
verbose_logger.info(
f"MCP list_tools - Successfully returned {len(tools)} tools"
@ -235,6 +238,7 @@ if MCP_AVAILABLE:
mcp_auth_header,
_,
mcp_server_auth_headers,
oauth2_headers,
) = get_auth_context()
verbose_logger.debug(
@ -266,6 +270,7 @@ if MCP_AVAILABLE:
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
**data, # for logging
)
except BlockedPiiEntityError as e:
@ -357,6 +362,7 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str],
mcp_servers: Optional[List[str]],
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
) -> List[MCPTool]:
"""
Helper method to fetch tools from MCP servers based on server filtering criteria.
@ -365,7 +371,8 @@ if MCP_AVAILABLE:
user_api_key_auth: User authentication info for access control
mcp_auth_header: Optional auth header for MCP server (deprecated)
mcp_servers: Optional list of server names/aliases to filter by
mcp_server_auth_headers: Optional dict of server-specific auth headers {server_alias: auth_value}
mcp_server_auth_headers: Optional dict of server-specific auth headers
oauth2_headers: Optional dict of oauth2 headers
Returns:
List[MCPTool]: Combined list of tools from filtered servers
@ -398,6 +405,10 @@ if MCP_AVAILABLE:
elif mcp_server_auth_headers and server.server_name is not None:
server_auth_header = mcp_server_auth_headers.get(server.server_name)
extra_headers: Optional[Dict[str, str]] = None
if server.auth_type == MCPAuth.oauth2:
extra_headers = oauth2_headers
# Fall back to deprecated mcp_auth_header if no server-specific header found
if server_auth_header is None:
server_auth_header = mcp_auth_header
@ -406,6 +417,7 @@ if MCP_AVAILABLE:
tools = await global_mcp_server_manager._get_tools_from_server(
server=server,
mcp_auth_header=server_auth_header,
extra_headers=extra_headers,
)
all_tools.extend(tools)
verbose_logger.debug(
@ -427,6 +439,7 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
) -> List[MCPTool]:
"""
List all available MCP tools.
@ -450,6 +463,7 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
)
verbose_logger.debug(
f"Successfully fetched {len(managed_tools)} tools from managed MCP servers"
@ -492,6 +506,7 @@ if MCP_AVAILABLE:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
**kwargs: Any,
) -> List[Union[TextContent, ImageContent, EmbeddedResource]]:
"""
@ -519,16 +534,16 @@ if MCP_AVAILABLE:
"litellm_logging_obj", None
)
if litellm_logging_obj:
litellm_logging_obj.model_call_details[
"mcp_tool_call_metadata"
] = standard_logging_mcp_tool_call
litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = (
standard_logging_mcp_tool_call
)
litellm_logging_obj.model = f"MCP: {name}"
# Try managed server tool first (pass the full prefixed name)
# Primary and recommended way to use MCP servers
#########################################################
mcp_server: Optional[
MCPServer
] = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
mcp_server: Optional[MCPServer] = (
global_mcp_server_manager._get_mcp_server_from_tool_name(name)
)
if mcp_server:
standard_logging_mcp_tool_call["mcp_server_cost_info"] = (
mcp_server.mcp_info or {}
@ -539,6 +554,7 @@ if MCP_AVAILABLE:
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
litellm_logging_obj=litellm_logging_obj,
)
@ -591,6 +607,7 @@ if MCP_AVAILABLE:
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
litellm_logging_obj: Optional[Any] = None,
) -> List[Union[TextContent, ImageContent, EmbeddedResource]]:
"""Handle tool execution for managed server tools"""
@ -603,6 +620,7 @@ if MCP_AVAILABLE:
user_api_key_auth=user_api_key_auth,
mcp_auth_header=mcp_auth_header,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
proxy_logging_obj=proxy_logging_obj,
)
verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result)
@ -638,26 +656,32 @@ if MCP_AVAILABLE:
mcp_path_match = re.match(r"^/mcp/([^?#]+)(?:\?.*)?(?:#.*)?$", path)
if mcp_path_match:
servers_and_path = mcp_path_match.group(1)
if servers_and_path:
# Check if it contains commas (comma-separated servers)
if ',' in servers_and_path:
if "," in servers_and_path:
# For comma-separated, look for a path at the end
# Common patterns: /tools, /chat/completions, etc.
path_match = re.search(r'/([^/,]+(?:/[^/,]+)*)$', servers_and_path)
path_match = re.search(r"/([^/,]+(?:/[^/,]+)*)$", servers_and_path)
if path_match:
# Path found at the end, remove it from servers
path_part = '/' + path_match.group(1)
servers_part = servers_and_path[:-len(path_part)]
mcp_servers_from_path = [s.strip() for s in servers_part.split(',') if s.strip()]
path_part = "/" + path_match.group(1)
servers_part = servers_and_path[: -len(path_part)]
mcp_servers_from_path = [
s.strip() for s in servers_part.split(",") if s.strip()
]
else:
# No path, just comma-separated servers
mcp_servers_from_path = [s.strip() for s in servers_and_path.split(',') if s.strip()]
mcp_servers_from_path = [
s.strip() for s in servers_and_path.split(",") if s.strip()
]
else:
# Single server case - use regex approach for server/path separation
# This handles cases like "custom_solutions/user_123/chat/completions"
# where we want to extract "custom_solutions/user_123" as the server name
single_server_match = re.match(r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", servers_and_path)
single_server_match = re.match(
r"^([^/]+(?:/[^/]+)?)(?:/.*)?$", servers_and_path
)
if single_server_match:
server_name = single_server_match.group(1)
mcp_servers_from_path = [server_name]
@ -677,6 +701,7 @@ if MCP_AVAILABLE:
mcp_auth_header,
_,
mcp_server_auth_headers,
oauth2_headers,
) = await MCPRequestHandler.process_mcp_request(scope)
mcp_servers = mcp_servers_from_path
else:
@ -685,8 +710,15 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
) = await MCPRequestHandler.process_mcp_request(scope)
return user_api_key_auth, mcp_auth_header, mcp_servers, mcp_server_auth_headers
return (
user_api_key_auth,
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
)
async def handle_streamable_http_mcp(
scope: Scope, receive: Receive, send: Send
@ -699,6 +731,7 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
) = await extract_mcp_auth_context(scope, path)
verbose_logger.debug(
f"MCP request mcp_servers (header/path): {mcp_servers}"
@ -712,6 +745,7 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
)
# Ensure session managers are initialized
@ -750,6 +784,7 @@ if MCP_AVAILABLE:
mcp_auth_header,
mcp_servers,
mcp_server_auth_headers,
oauth2_headers,
) = await extract_mcp_auth_context(scope, path)
verbose_logger.debug(
f"MCP request mcp_servers (header/path): {mcp_servers}"
@ -762,6 +797,7 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
)
if not _SESSION_MANAGERS_INITIALIZED:
@ -809,6 +845,8 @@ if MCP_AVAILABLE:
# Mount the MCP handlers
app.mount("/", handle_streamable_http_mcp)
app.mount("/mcp", handle_streamable_http_mcp)
app.mount("/{mcp_server_name}/mcp", handle_streamable_http_mcp)
app.mount("/sse", handle_sse_mcp)
app.add_middleware(AuthContextMiddleware)
@ -821,6 +859,7 @@ if MCP_AVAILABLE:
mcp_auth_header: Optional[str] = None,
mcp_servers: Optional[List[str]] = None,
mcp_server_auth_headers: Optional[Dict[str, str]] = None,
oauth2_headers: Optional[Dict[str, str]] = None,
) -> None:
"""
Set the UserAPIKeyAuth in the auth context variable.
@ -836,17 +875,17 @@ if MCP_AVAILABLE:
mcp_auth_header=mcp_auth_header,
mcp_servers=mcp_servers,
mcp_server_auth_headers=mcp_server_auth_headers,
oauth2_headers=oauth2_headers,
)
auth_context_var.set(auth_user)
def get_auth_context() -> (
Tuple[
Optional[UserAPIKeyAuth],
Optional[str],
Optional[List[str]],
Optional[Dict[str, str]],
]
):
def get_auth_context() -> Tuple[
Optional[UserAPIKeyAuth],
Optional[str],
Optional[List[str]],
Optional[Dict[str, str]],
Optional[Dict[str, str]],
]:
"""
Get the UserAPIKeyAuth from the auth context variable.
@ -861,8 +900,9 @@ if MCP_AVAILABLE:
auth_user.mcp_auth_header,
auth_user.mcp_servers,
auth_user.mcp_server_auth_headers,
auth_user.oauth2_headers,
)
return None, None, None, None
return None, None, None, None, None
########################################################
############ End of Auth Context Functions #############

0
litellm/proxy/_experimental/out/_next/static/0oPk2eYtSaTLaPyVixqA8/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/Ap4Kq4vtq74RgOyxD-zii/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/0oPk2eYtSaTLaPyVixqA8/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/Ap4Kq4vtq74RgOyxD-zii/_ssgManifest.js
View file

2
litellm/proxy/_experimental/out/_next/static/chunks/162-4e7640b4d68e1ae4.js → litellm/proxy/_experimental/out/_next/static/chunks/162-dd6427ff1a4ad9f4.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/172-0f7049c565983c4d.js
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/_next/static/chunks/220-89d73a525e307735.js → litellm/proxy/_experimental/out/_next/static/chunks/313-0025fb08e386c4b8.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/851-bbe6d02cf41bb87a.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-304b7041a3fa39f7.js → litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-0b693f691bf0309f.js
View file

@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{38520:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=38520)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{38520:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(97851);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,851,971,117,744],function(){return e(e.s=38520)}),_N_E=e.O()}]);

1
litellm/proxy/_experimental/out/_next/static/chunks/app/page-46f79791404274c7.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/app/page-73b19c9fbf8cc64f.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/assets/logos/oracle.svg Normal file
View file

@ -0,0 +1 @@
<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 231 30' preserveAspectRatio='xMinYMid'><path d='M99.61,19.52h15.24l-8.05-13L92,30H85.27l18-28.17a4.29,4.29,0,0,1,7-.05L128.32,30h-6.73l-3.17-5.25H103l-3.36-5.23m69.93,5.23V0.28h-5.72V27.16a2.76,2.76,0,0,0,.85,2,2.89,2.89,0,0,0,2.08.87h26l3.39-5.25H169.54M75,20.38A10,10,0,0,0,75,.28H50V30h5.71V5.54H74.65a4.81,4.81,0,0,1,0,9.62H58.54L75.6,30h8.29L72.43,20.38H75M14.88,30H32.15a14.86,14.86,0,0,0,0-29.71H14.88a14.86,14.86,0,1,0,0,29.71m16.88-5.23H15.26a9.62,9.62,0,0,1,0-19.23h16.5a9.62,9.62,0,1,1,0,19.23M140.25,30h17.63l3.34-5.23H140.64a9.62,9.62,0,1,1,0-19.23h16.75l3.38-5.25H140.25a14.86,14.86,0,1,0,0,29.71m69.87-5.23a9.62,9.62,0,0,1-9.26-7h24.42l3.36-5.24H200.86a9.61,9.61,0,0,1,9.26-7h16.76l3.35-5.25h-20.5a14.86,14.86,0,0,0,0,29.71h17.63l3.35-5.23h-20.6' transform='translate(-0.02 0)' style='fill:#C74634'/></svg>
Side by side

After

Width:  |  Height:  |  Size: 874 B

2
litellm/proxy/_experimental/out/index.html
View file

File diff suppressed because one or more lines are too long

4
litellm/proxy/_experimental/out/index.txt
View file

@ -1,7 +1,7 @@
2:I[19107,[],"ClientPageRoot"]
3:I[85617,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","220","static/chunks/220-89d73a525e307735.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-4e7640b4d68e1ae4.js","172","static/chunks/172-0f7049c565983c4d.js","931","static/chunks/app/page-73b19c9fbf8cc64f.js"],"default",1]
3:I[55139,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","313","static/chunks/313-0025fb08e386c4b8.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-dd6427ff1a4ad9f4.js","851","static/chunks/851-bbe6d02cf41bb87a.js","931","static/chunks/app/page-46f79791404274c7.js"],"default",1]
4:I[4707,[],""]
5:I[36423,[],""]
0:["0oPk2eYtSaTLaPyVixqA8",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
0:["Ap4Kq4vtq74RgOyxD-zii",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
1:null

4
litellm/proxy/_experimental/out/model_hub.txt
View file

@ -1,7 +1,7 @@
2:I[19107,[],"ClientPageRoot"]
3:I[52829,["50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-4e7640b4d68e1ae4.js","418","static/chunks/app/model_hub/page-13b00ef4a072d920.js"],"default",1]
3:I[52829,["50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-dd6427ff1a4ad9f4.js","418","static/chunks/app/model_hub/page-13b00ef4a072d920.js"],"default",1]
4:I[4707,[],""]
5:I[36423,[],""]
0:["0oPk2eYtSaTLaPyVixqA8",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
0:["Ap4Kq4vtq74RgOyxD-zii",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
1:null

2
litellm/proxy/_experimental/out/model_hub_table/index.html → litellm/proxy/_experimental/out/model_hub_table.html
View file

File diff suppressed because one or more lines are too long

4
litellm/proxy/_experimental/out/model_hub_table.txt
View file

@ -1,7 +1,7 @@
2:I[19107,[],"ClientPageRoot"]
3:I[22775,["50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-4e7640b4d68e1ae4.js","172","static/chunks/172-0f7049c565983c4d.js","25","static/chunks/app/model_hub_table/page-304b7041a3fa39f7.js"],"default",1]
3:I[22775,["50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-dd6427ff1a4ad9f4.js","851","static/chunks/851-bbe6d02cf41bb87a.js","25","static/chunks/app/model_hub_table/page-0b693f691bf0309f.js"],"default",1]
4:I[4707,[],""]
5:I[36423,[],""]
0:["0oPk2eYtSaTLaPyVixqA8",[[["",{"children":["model_hub_table",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub_table",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub_table","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
0:["Ap4Kq4vtq74RgOyxD-zii",[[["",{"children":["model_hub_table",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub_table",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub_table","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
1:null

1
litellm/proxy/_experimental/out/onboarding.html Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/onboarding.txt
View file

@ -2,6 +2,6 @@
3:I[12011,["665","static/chunks/3014691f-b7b79b78e27792f3.js","50","static/chunks/50-d0da2dd7acce2eb9.js","154","static/chunks/154-b1f2a106d0e0d77b.js","461","static/chunks/app/onboarding/page-d0d85032bb87ba51.js"],"default",1]
4:I[4707,[],""]
5:I[36423,[],""]
0:["0oPk2eYtSaTLaPyVixqA8",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
0:["Ap4Kq4vtq74RgOyxD-zii",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
1:null

13
litellm/proxy/_new_secret_config.yaml
View file

@ -15,3 +15,16 @@ model_list:
model: hosted_vllm/whisper-v3
api_base: "https://webhook.site/2f385e05-00aa-402b-86d1-efc9261471a5"
api_key: dummy
mcp_servers:
github_mcp:
url: "https://api.githubcopilot.com/mcp"
auth_type: oauth2
authorization_url: https://github.com/login/oauth/authorize
token_url: https://github.com/login/oauth/access_token
client_id: os.environ/GITHUB_OAUTH_CLIENT_ID
client_secret: os.environ/GITHUB_OAUTH_CLIENT_SECRET
scopes: ["public_repo", "user:email"]
# allowed_tools: ["list_tools"]
# disallowed_tools: ["repo_delete"]

133
litellm/proxy/_types.py
View file

@ -1,16 +1,7 @@
import enum
import json
from datetime import datetime
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
List,
Literal,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Union
import httpx
from pydantic import (
@ -26,11 +17,7 @@ from typing_extensions import Required, TypedDict
from litellm._uuid import uuid
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.llms.openai import AllMessageValues, OpenAIFileObject
from litellm.types.mcp import (
MCPAuthType,
MCPTransport,
MCPTransportType,
)
from litellm.types.mcp import MCPAuthType, MCPTransport, MCPTransportType
from litellm.types.mcp_server.mcp_server_manager import MCPInfo
from litellm.types.router import RouterErrors, UpdateRouterConfig
from litellm.types.secret_managers.main import KeyManagementSystem
@ -404,16 +391,16 @@ class LiteLLMRoutes(enum.Enum):
]
key_management_routes = [
KeyManagementRoutes.KEY_GENERATE,
KeyManagementRoutes.KEY_UPDATE,
KeyManagementRoutes.KEY_DELETE,
KeyManagementRoutes.KEY_INFO,
KeyManagementRoutes.KEY_REGENERATE,
KeyManagementRoutes.KEY_GENERATE_SERVICE_ACCOUNT,
KeyManagementRoutes.KEY_REGENERATE_WITH_PATH_PARAM,
KeyManagementRoutes.KEY_LIST,
KeyManagementRoutes.KEY_BLOCK,
KeyManagementRoutes.KEY_UNBLOCK,
KeyManagementRoutes.KEY_GENERATE.value,
KeyManagementRoutes.KEY_UPDATE.value,
KeyManagementRoutes.KEY_DELETE.value,
KeyManagementRoutes.KEY_INFO.value,
KeyManagementRoutes.KEY_REGENERATE.value,
KeyManagementRoutes.KEY_GENERATE_SERVICE_ACCOUNT.value,
KeyManagementRoutes.KEY_REGENERATE_WITH_PATH_PARAM.value,
KeyManagementRoutes.KEY_LIST.value,
KeyManagementRoutes.KEY_BLOCK.value,
KeyManagementRoutes.KEY_UNBLOCK.value,
]
management_routes = [
@ -747,9 +734,9 @@ class GenerateRequestBase(LiteLLMPydanticObjectBase):
allowed_cache_controls: Optional[list] = []
config: Optional[dict] = {}
permissions: Optional[dict] = {}
model_max_budget: Optional[
dict
] = {} # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {}
model_max_budget: Optional[dict] = (
{}
) # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {}
model_config = ConfigDict(protected_namespaces=())
model_rpm_limit: Optional[dict] = None
@ -788,12 +775,11 @@ class GenerateKeyRequest(KeyRequestBase):
description="Type of key that determines default allowed routes.",
)
auto_rotate: Optional[bool] = Field(
default=False,
description="Whether this key should be automatically rotated"
default=False, description="Whether this key should be automatically rotated"
)
rotation_interval: Optional[str] = Field(
default=None,
description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True"
description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True",
)
@ -842,6 +828,8 @@ class UpdateKeyRequest(KeyRequestBase):
metadata: Optional[dict] = None
temp_budget_increase: Optional[float] = None
temp_budget_expiry: Optional[datetime] = None
auto_rotate: Optional[bool] = None
rotation_interval: Optional[str] = None
@model_validator(mode="after")
def validate_temp_budget(self) -> "UpdateKeyRequest":
@ -1155,12 +1143,12 @@ class NewCustomerRequest(BudgetNewRequest):
blocked: bool = False # allow/disallow requests for this end-user
budget_id: Optional[str] = None # give either a budget_id or max_budget
spend: Optional[float] = None
allowed_model_region: Optional[
AllowedModelRegion
] = None # require all user requests to use models in this specific region
default_model: Optional[
str
] = None # if no equivalent model in allowed region - default all requests to this model
allowed_model_region: Optional[AllowedModelRegion] = (
None # require all user requests to use models in this specific region
)
default_model: Optional[str] = (
None # if no equivalent model in allowed region - default all requests to this model
)
@model_validator(mode="before")
@classmethod
@ -1182,12 +1170,12 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase):
blocked: bool = False # allow/disallow requests for this end-user
max_budget: Optional[float] = None
budget_id: Optional[str] = None # give either a budget_id or max_budget
allowed_model_region: Optional[
AllowedModelRegion
] = None # require all user requests to use models in this specific region
default_model: Optional[
str
] = None # if no equivalent model in allowed region - default all requests to this model
allowed_model_region: Optional[AllowedModelRegion] = (
None # require all user requests to use models in this specific region
)
default_model: Optional[str] = (
None # if no equivalent model in allowed region - default all requests to this model
)
class DeleteCustomerRequest(LiteLLMPydanticObjectBase):
@ -1261,15 +1249,15 @@ class NewTeamRequest(TeamBase):
guardrails: Optional[List[str]] = None
prompts: Optional[List[str]] = None
object_permission: Optional[LiteLLM_ObjectPermissionBase] = None
team_member_budget: Optional[
float
] = None # allow user to set a budget for all team members
team_member_rpm_limit: Optional[
int
] = None # allow user to set RPM limit for all team members
team_member_tpm_limit: Optional[
int
] = None # allow user to set TPM limit for all team members
team_member_budget: Optional[float] = (
None # allow user to set a budget for all team members
)
team_member_rpm_limit: Optional[int] = (
None # allow user to set RPM limit for all team members
)
team_member_tpm_limit: Optional[int] = (
None # allow user to set TPM limit for all team members
)
team_member_key_duration: Optional[str] = None # e.g. "1d", "1w", "1m"
model_config = ConfigDict(protected_namespaces=())
@ -1348,9 +1336,9 @@ class BlockKeyRequest(LiteLLMPydanticObjectBase):
class AddTeamCallback(LiteLLMPydanticObjectBase):
callback_name: str
callback_type: Optional[
Literal["success", "failure", "success_and_failure"]
] = "success_and_failure"
callback_type: Optional[Literal["success", "failure", "success_and_failure"]] = (
"success_and_failure"
)
callback_vars: Dict[str, str]
@model_validator(mode="before")
@ -1619,9 +1607,9 @@ class ConfigList(LiteLLMPydanticObjectBase):
stored_in_db: Optional[bool]
field_default_value: Any
premium_field: bool = False
nested_fields: Optional[
List[FieldDetail]
] = None # For nested dictionary or Pydantic fields
nested_fields: Optional[List[FieldDetail]] = (
None # For nested dictionary or Pydantic fields
)
class UserHeaderMapping(LiteLLMPydanticObjectBase):
@ -1814,6 +1802,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase):
auto_rotate: Optional[bool] = False # Whether this key should be auto-rotated
rotation_interval: Optional[str] = None # How often to rotate (e.g., "30d", "90d")
last_rotation_at: Optional[datetime] = None # When this key was last rotated
key_rotation_at: Optional[datetime] = None # When this key should next be rotated
model_config = ConfigDict(protected_namespaces=())
@ -1928,7 +1917,7 @@ class UserAPIKeyAuth(
key_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME,
team_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME,
)
@classmethod
def get_litellm_cli_user_api_key_auth(cls) -> "UserAPIKeyAuth":
"""
@ -1944,7 +1933,7 @@ class UserAPIKeyAuth(
key_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME,
team_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME,
)
@classmethod
def get_litellm_internal_jobs_user_api_key_auth(cls) -> "UserAPIKeyAuth":
"""
@ -1987,9 +1976,9 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase):
budget_id: Optional[str] = None
created_at: datetime
updated_at: datetime
user: Optional[
Any
] = None # You might want to replace 'Any' with a more specific type if available
user: Optional[Any] = (
None # You might want to replace 'Any' with a more specific type if available
)
litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
model_config = ConfigDict(protected_namespaces=())
@ -2884,9 +2873,9 @@ class TeamModelDeleteRequest(BaseModel):
# Organization Member Requests
class OrganizationMemberAddRequest(OrgMemberAddRequest):
organization_id: str
max_budget_in_organization: Optional[
float
] = None # Users max budget within the organization
max_budget_in_organization: Optional[float] = (
None # Users max budget within the organization
)
class OrganizationMemberDeleteRequest(MemberDeleteRequest):
@ -3096,9 +3085,9 @@ class ProviderBudgetResponse(LiteLLMPydanticObjectBase):
Maps provider names to their budget configs.
"""
providers: Dict[
str, ProviderBudgetResponseObject
] = {} # Dictionary mapping provider names to their budget configurations
providers: Dict[str, ProviderBudgetResponseObject] = (
{}
) # Dictionary mapping provider names to their budget configurations
class ProxyStateVariables(TypedDict):
@ -3232,9 +3221,9 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
enforce_rbac: bool = False
roles_jwt_field: Optional[str] = None # v2 on role mappings
role_mappings: Optional[List[RoleMapping]] = None
object_id_jwt_field: Optional[
str
] = None # can be either user / team, inferred from the role mapping
object_id_jwt_field: Optional[str] = (
None # can be either user / team, inferred from the role mapping
)
scope_mappings: Optional[List[ScopeMapping]] = None
enforce_scope_based_access: bool = False
enforce_team_based_model_access: bool = False

4
litellm/proxy/auth/oauth2_proxy_hook.py
View file

@ -14,8 +14,8 @@ async def handle_oauth2_proxy_request(request: Request) -> UserAPIKeyAuth:
verbose_proxy_logger.debug("Handling oauth2 proxy request")
# Define the OAuth2 config mappings
oauth2_config_mappings: Dict[str, str] = general_settings.get(
"oauth2_config_mappings", None
oauth2_config_mappings: Dict[str, str] = (
general_settings.get("oauth2_config_mappings") or {}
)
verbose_proxy_logger.debug(f"Oauth2 config mappings: {oauth2_config_mappings}")

25
litellm/proxy/auth/user_api_key_auth.py
View file

@ -390,7 +390,6 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
pass_through_endpoints: Optional[List[dict]] = general_settings.get(
"pass_through_endpoints", None
)
passed_in_key: Optional[str] = None
## CHECK IF X-LITELM-API-KEY IS PASSED IN - supercedes Authorization header
api_key, passed_in_key = get_api_key(
custom_litellm_key_header=custom_litellm_key_header,
@ -502,7 +501,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
end_user_object = result["end_user_object"]
org_id = result["org_id"]
token = result["token"]
team_membership: Optional[LiteLLM_TeamMembership] = result.get("team_membership", None)
team_membership: Optional[LiteLLM_TeamMembership] = result.get(
"team_membership", None
)
global_proxy_spend = await get_global_proxy_spend(
litellm_proxy_admin_name=litellm_proxy_admin_name,
@ -537,10 +538,22 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
org_id=org_id,
parent_otel_span=parent_otel_span,
end_user_id=end_user_id,
user_tpm_limit=user_object.tpm_limit if user_object is not None else None,
user_rpm_limit=user_object.rpm_limit if user_object is not None else None,
team_member_rpm_limit=team_membership.safe_get_team_member_rpm_limit() if team_membership is not None else None,
team_member_tpm_limit=team_membership.safe_get_team_member_tpm_limit() if team_membership is not None else None,
user_tpm_limit=(
user_object.tpm_limit if user_object is not None else None
),
user_rpm_limit=(
user_object.rpm_limit if user_object is not None else None
),
team_member_rpm_limit=(
team_membership.safe_get_team_member_rpm_limit()
if team_membership is not None
else None
),
team_member_tpm_limit=(
team_membership.safe_get_team_member_tpm_limit()
if team_membership is not None
else None
),
)
# run through common checks
_ = await common_checks(

49
litellm/proxy/common_utils/key_rotation_manager.py
View file

@ -4,7 +4,7 @@ Key Rotation Manager - Automated key rotation based on rotation schedules
Handles finding keys that need rotation based on their individual schedules.
"""
from datetime import datetime, timedelta, timezone
from datetime import datetime, timezone
from typing import List
from litellm._logging import verbose_proxy_logger
@ -16,6 +16,7 @@ from litellm.proxy._types import (
)
from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
from litellm.proxy.management_endpoints.key_management_endpoints import (
_calculate_key_rotation_time,
regenerate_key_fn,
)
from litellm.proxy.utils import PrismaClient
@ -60,49 +61,39 @@ class KeyRotationManager:
async def _find_keys_needing_rotation(self) -> List[LiteLLM_VerificationToken]:
"""
Find keys that are due for rotation based on their rotation interval.
Find keys that are due for rotation based on their key_rotation_at timestamp.
Logic:
- Key has auto_rotate = true
- Key has rotation_interval set
- Either: never been rotated (last_rotation_at is null) OR
- Time since last rotation >= rotation_interval
- key_rotation_at is null (needs initial setup) OR key_rotation_at <= now
"""
now = datetime.now(timezone.utc)
keys_with_rotation = await self.prisma_client.db.litellm_verificationtoken.find_many(
where={
"auto_rotate": True, # Only keys marked for auto rotation
"rotation_interval": {"not": None} # Must have rotation interval set
"OR": [
{"key_rotation_at": None}, # Keys that need initial rotation time setup
{"key_rotation_at": {"lte": now}} # Keys where rotation time has passed
]
}
)
# Filter keys that need rotation based on last_rotation_at + interval
keys_needing_rotation = []
now = datetime.now(timezone.utc)
for key in keys_with_rotation:
if self._should_rotate_key(key, now):
keys_needing_rotation.append(key)
return keys_needing_rotation
return keys_with_rotation
def _should_rotate_key(self, key: LiteLLM_VerificationToken, now: datetime) -> bool:
"""
Determine if a key should be rotated based on last rotation time and interval.
Determine if a key should be rotated based on key_rotation_at timestamp.
"""
if not key.rotation_interval:
return False
# If never rotated, rotate immediately
if key.last_rotation_at is None:
# If key_rotation_at is not set, rotate immediately (and set it)
if key.key_rotation_at is None:
return True
# Calculate if enough time has passed since last rotation
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
interval_seconds = duration_in_seconds(key.rotation_interval)
next_rotation_time = key.last_rotation_at + timedelta(seconds=interval_seconds)
return now >= next_rotation_time
# Check if the rotation time has passed
return now >= key.key_rotation_at
async def _rotate_key(self, key: LiteLLM_VerificationToken):
"""
@ -125,12 +116,16 @@ class KeyRotationManager:
)
# Update the NEW key with rotation info (regenerate_key_fn creates a new token)
if isinstance(response, GenerateKeyResponse) and response.token_id:
if isinstance(response, GenerateKeyResponse) and response.token_id and key.rotation_interval:
# Calculate next rotation time using helper function
now = datetime.now(timezone.utc)
next_rotation_time = _calculate_key_rotation_time(key.rotation_interval)
await self.prisma_client.db.litellm_verificationtoken.update(
where={"token": response.token_id},
data={
"rotation_count": (key.rotation_count or 0) + 1,
"last_rotation_at": datetime.now(timezone.utc)
"last_rotation_at": now,
"key_rotation_at": next_rotation_time
}
)

38
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -22,9 +22,7 @@ from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching import DualCache
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
@ -363,7 +361,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
prepared_request.headers,
)
response = await self.async_handler.post(
httpx_response = await self.async_handler.post(
url=prepared_request.url,
data=prepared_request.body, # type: ignore
headers=prepared_request.headers, # type: ignore
@ -373,19 +371,19 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
self.add_standard_logging_guardrail_information_to_request_data(
guardrail_provider=self.guardrail_provider,
guardrail_json_response=response.json(),
guardrail_json_response=httpx_response.json(),
request_data=request_data or {},
guardrail_status=self._get_bedrock_guardrail_response_status(
response=response
response=httpx_response
),
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
)
#########################################################
if response.status_code == 200:
if httpx_response.status_code == 200:
# check if the response was flagged
_json_response = response.json()
_json_response = httpx_response.json()
redacted_response = _redact_pii_matches(_json_response)
verbose_proxy_logger.debug("Bedrock AI response : %s", redacted_response)
bedrock_guardrail_response = BedrockGuardrailResponse(**_json_response)
@ -398,8 +396,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
else:
verbose_proxy_logger.error(
"Bedrock AI: error in response. Status code: %s, response: %s",
response.status_code,
response.text,
httpx_response.status_code,
httpx_response.text,
)
return bedrock_guardrail_response
@ -597,11 +595,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
########## 2. Update the messages with the guardrail response ##########
#########################################################
data[
"messages"
] = self._update_messages_with_updated_bedrock_guardrail_response(
messages=new_messages,
bedrock_guardrail_response=bedrock_guardrail_response,
data["messages"] = (
self._update_messages_with_updated_bedrock_guardrail_response(
messages=new_messages,
bedrock_guardrail_response=bedrock_guardrail_response,
)
)
#########################################################
@ -652,11 +650,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
########## 2. Update the messages with the guardrail response ##########
#########################################################
data[
"messages"
] = self._update_messages_with_updated_bedrock_guardrail_response(
messages=new_messages,
bedrock_guardrail_response=bedrock_guardrail_response,
data["messages"] = (
self._update_messages_with_updated_bedrock_guardrail_response(
messages=new_messages,
bedrock_guardrail_response=bedrock_guardrail_response,
)
)
#########################################################

168
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -292,6 +292,70 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return RateLimitResponse(overall_code=overall_code, statuses=statuses)
def _group_keys_by_hash_tag(self, keys: List[str]) -> Dict[str, List[str]]:
"""
Group keys by their Redis hash tag to ensure cluster compatibility.
Keys with the same hash tag will be processed together.
"""
groups: Dict[str, List[str]] = {}
for key in keys:
# Extract hash tag from key like "{api_key:sk-123}:requests"
if "{" in key and "}" in key:
start = key.find("{")
end = key.find("}", start)
hash_tag = key[start : end + 1]
else:
# Fallback for keys without hash tags
hash_tag = "no_hash_tag"
if hash_tag not in groups:
groups[hash_tag] = []
groups[hash_tag].append(key)
return groups
async def _execute_redis_batch_rate_limiter_script(
self,
keys_to_fetch: List[str],
now_int: int,
) -> List[Any]:
"""
Execute Redis operations grouped by hash tag for cluster compatibility.
Args:
keys_to_fetch: List[str] - List of keys to fetch
now_int: int - Current timestamp
Returns:
List[Any] - List of cache values
"""
if self.batch_rate_limiter_script is None:
return []
key_groups = self._group_keys_by_hash_tag(keys_to_fetch)
all_cache_values = []
for hash_tag, group_keys in key_groups.items():
try:
group_cache_values = await self.batch_rate_limiter_script(
keys=group_keys,
args=[now_int, self.window_size], # Use integer timestamp
)
all_cache_values.extend(group_cache_values)
except Exception as e:
verbose_proxy_logger.warning(
f"Redis Lua script failed for hash tag {hash_tag}: {str(e)}"
)
# Fallback to in-memory cache for this group
group_cache_values = await self.in_memory_cache_sliding_window(
keys=group_keys,
now_int=now_int,
window_size=self.window_size,
)
all_cache_values.extend(group_cache_values)
return all_cache_values
async def should_rate_limit(
self,
descriptors: List[RateLimitDescriptor],
@ -313,7 +377,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
for descriptor in descriptors:
descriptor_key = descriptor["key"]
descriptor_value = descriptor["value"]
rate_limit = descriptor.get("rate_limit", {}) or {}
rate_limit: RateLimitDescriptorRateLimitObject = (
descriptor.get("rate_limit") or RateLimitDescriptorRateLimitObject()
)
requests_limit = rate_limit.get("requests_per_unit")
tokens_limit = rate_limit.get("tokens_per_unit")
max_parallel_requests_limit = rate_limit.get("max_parallel_requests")
@ -374,9 +440,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
## IF under limit, check Redis
if self.batch_rate_limiter_script is not None:
cache_values = await self.batch_rate_limiter_script(
keys=keys_to_fetch,
args=[now_int, self.window_size], # Use integer timestamp
# Group keys by hash tag for Redis cluster compatibility
cache_values = await self._execute_redis_batch_rate_limiter_script(
keys_to_fetch=keys_to_fetch,
now_int=now_int,
)
# update in-memory cache with new values
@ -566,26 +633,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
for i, status in enumerate(response["statuses"]):
if status["code"] == "OVER_LIMIT":
descriptor = descriptors[floor(i / 2)]
# Calculate reset time (window_start + window_size)
now = datetime.now().timestamp()
reset_time = now + self.window_size # Conservative estimate
reset_time_formatted = datetime.fromtimestamp(reset_time).strftime("%Y-%m-%d %H:%M:%S UTC")
reset_time_formatted = datetime.fromtimestamp(
reset_time
).strftime("%Y-%m-%d %H:%M:%S UTC")
# Handle negative remaining values more gracefully
remaining_display = max(0, status['limit_remaining'])
remaining_display = max(0, status["limit_remaining"])
# Create detailed error message
rate_limit_type = status['rate_limit_type']
current_limit = status['current_limit']
rate_limit_type = status["rate_limit_type"]
current_limit = status["current_limit"]
detail = (
f"Rate limit exceeded for {descriptor['key']}: {descriptor['value']}. "
f"Limit type: {rate_limit_type}. "
f"Current limit: {current_limit}, Remaining: {remaining_display}. "
f"Limit resets at: {reset_time_formatted}"
)
raise HTTPException(
status_code=429,
detail=detail,
@ -628,6 +697,45 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return pipeline_operations
async def _execute_token_increment_script(
self,
pipeline_operations: List["RedisPipelineIncrementOperation"],
) -> None:
"""
Execute token increment script grouped by hash tag for cluster compatibility.
"""
if self.token_increment_script is None:
return
# Group operations by hash tag for Redis cluster compatibility
operation_keys = [op["key"] for op in pipeline_operations]
key_groups = self._group_keys_by_hash_tag(operation_keys)
for _hash_tag, group_keys in key_groups.items():
# Get operations for this hash tag group
group_operations = [
op for op in pipeline_operations if op["key"] in group_keys
]
keys = []
args = []
for op in group_operations:
# Convert None TTL to 0 for Lua script
ttl_value = op["ttl"] if op["ttl"] is not None else 0
verbose_proxy_logger.debug(
f"Executing TTL-preserving increment for key={op['key']}, "
f"increment={op['increment_value']}, ttl={ttl_value}"
)
keys.append(op["key"])
args.extend([op["increment_value"], ttl_value])
await self.token_increment_script(
keys=keys,
args=args,
)
async def async_increment_tokens_with_ttl_preservation(
self,
pipeline_operations: List["RedisPipelineIncrementOperation"],
@ -652,25 +760,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
return
try:
# Use Lua script for all operations
keys = []
args = []
for op in pipeline_operations:
# Convert None TTL to 0 for Lua script
ttl_value = op["ttl"] if op["ttl"] is not None else 0
verbose_proxy_logger.debug(
f"Executing TTL-preserving increment for key={op['key']}, "
f"increment={op['increment_value']}, ttl={ttl_value}"
)
keys.append(op["key"])
args.extend([op["increment_value"], ttl_value])
await self.token_increment_script(
keys=keys,
args=args,
)
await self._execute_token_increment_script(pipeline_operations)
verbose_proxy_logger.debug(
f"Successfully executed TTL-preserving increment for {len(pipeline_operations)} keys"
@ -708,8 +798,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
_get_parent_otel_span_from_kwargs,
)
from litellm.proxy.common_utils.callback_utils import (
get_metadata_variable_name_from_kwargs,
get_model_group_from_litellm_kwargs,
get_metadata_variable_name_from_kwargs
)
from litellm.types.caching import RedisPipelineIncrementOperation
from litellm.types.utils import ModelResponse, Usage
@ -725,7 +815,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
# Get metadata from kwargs
litellm_metadata = kwargs["litellm_params"].get(get_metadata_variable_name_from_kwargs(kwargs), {})
litellm_metadata = kwargs["litellm_params"].get(
get_metadata_variable_name_from_kwargs(kwargs), {}
)
if litellm_metadata is None:
return
user_api_key = litellm_metadata.get("user_api_key")
@ -739,7 +831,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# Get total tokens from response
total_tokens = 0
# spot fix for /responses api
if (isinstance(response_obj, ModelResponse) or isinstance(response_obj, BaseLiteLLMOpenAIResponseObject)):
if isinstance(response_obj, ModelResponse) or isinstance(
response_obj, BaseLiteLLMOpenAIResponseObject
):
_usage = getattr(response_obj, "usage", None)
if _usage and isinstance(_usage, Usage):
if rate_limit_type == "output":
@ -857,7 +951,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
_get_parent_otel_span_from_kwargs(kwargs)
)
litellm_metadata = kwargs["litellm_params"]["metadata"]
user_api_key = litellm_metadata.get("user_api_key") if litellm_metadata else None
user_api_key = (
litellm_metadata.get("user_api_key") if litellm_metadata else None
)
pipeline_operations: List[RedisPipelineIncrementOperation] = []
if user_api_key:

52
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -87,6 +87,38 @@ def _get_user_in_team(
return None
def _calculate_key_rotation_time(rotation_interval: str) -> datetime:
"""
Helper function to calculate the next rotation time for a key based on the rotation interval.
Args:
rotation_interval: String representing the rotation interval (e.g., '30d', '90d', '1h')
Returns:
datetime: The calculated next rotation time in UTC
"""
now = datetime.now(timezone.utc)
interval_seconds = duration_in_seconds(rotation_interval)
return now + timedelta(seconds=interval_seconds)
def _set_key_rotation_fields(data: dict, auto_rotate: bool, rotation_interval: Optional[str]) -> None:
"""
Helper function to set rotation fields in key data if auto_rotate is enabled.
Args:
data: Dictionary to update with rotation fields
auto_rotate: Whether auto rotation is enabled
rotation_interval: The rotation interval string (required if auto_rotate is True)
"""
if auto_rotate and rotation_interval:
data.update({
"auto_rotate": auto_rotate,
"rotation_interval": rotation_interval,
"key_rotation_at": _calculate_key_rotation_time(rotation_interval)
})
def _is_allowed_to_make_key_request(
user_api_key_dict: UserAPIKeyAuth, user_id: Optional[str], team_id: Optional[str]
) -> bool:
@ -1071,6 +1103,8 @@ async def update_key_fn(
- allowed_routes: Optional[list] - List of allowed routes for the key. Store the actual route or store a wildcard pattern for a set of routes. Example - ["/chat/completions", "/embeddings", "/keys/*"]
- prompts: Optional[List[str]] - List of allowed prompts for the key. If specified, the key will only be able to use these specific prompts.
- object_permission: Optional[LiteLLM_ObjectPermissionBase] - key-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"]}. IF null or {} then no object permission.
- auto_rotate: Optional[bool] - Whether this key should be automatically rotated
- rotation_interval: Optional[str] - How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True
Example:
```bash
curl --location 'http://0.0.0.0:4000/key/update' \
@ -1162,6 +1196,13 @@ async def update_key_fn(
existing_key_token=existing_key_row.token,
)
# Handle rotation fields if auto_rotate is being enabled
_set_key_rotation_fields(
non_default_values,
non_default_values.get("auto_rotate", False),
non_default_values.get("rotation_interval")
)
_data = {**non_default_values, "token": key}
response = await prisma_client.update_data(token=key, data=_data)
@ -1727,12 +1768,11 @@ async def generate_key_helper_fn( # noqa: PLR0915
}
# Add rotation fields if auto_rotate is enabled
if auto_rotate and rotation_interval:
key_data.update({
"auto_rotate": auto_rotate,
"rotation_interval": rotation_interval
# last_rotation_at will be null initially - rotation happens on first check
})
_set_key_rotation_fields(
data=key_data,
auto_rotate=auto_rotate or False,
rotation_interval=rotation_interval
)
if (
get_secret("DISABLE_KEY_NAME", False) is True

123
litellm/proxy/management_endpoints/ui_sso.py
View file

@ -10,7 +10,6 @@ Has all /sso/* routes
import asyncio
import os
from litellm._uuid import uuid
from copy import deepcopy
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
@ -19,6 +18,7 @@ from fastapi.responses import RedirectResponse
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.caching import DualCache
from litellm.constants import MAX_SPENDLOG_ROWS_TO_QUERY
from litellm.llms.custom_httpx.http_handler import (
@ -115,7 +115,10 @@ def process_sso_jwt_access_token(
@router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False)
async def google_login(
request: Request, source: Optional[str] = None, key: Optional[str] = None, existing_key: Optional[str] = None
request: Request,
source: Optional[str] = None,
key: Optional[str] = None,
existing_key: Optional[str] = None,
): # noqa: PLR0915
"""
Create Proxy API Keys using Google Workspace SSO. Requires setting PROXY_BASE_URL in .env
@ -664,17 +667,20 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa:
status_code=401,
detail="Result not returned by SSO provider.",
)
if state and state.startswith(f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:"):
# Extract the key ID from the state
key_id = state.split(":", 1)[1]
# Get existing_key from query parameters if provided
existing_key = request.query_params.get("existing_key")
verbose_proxy_logger.info(f"CLI SSO callback detected for key: {key_id}, existing_key: {existing_key}")
return await cli_sso_callback(request=request, key=key_id, existing_key=existing_key, result=result)
verbose_proxy_logger.info(
f"CLI SSO callback detected for key: {key_id}, existing_key: {existing_key}"
)
return await cli_sso_callback(
request=request, key=key_id, existing_key=existing_key, result=result
)
return await SSOAuthenticationHandler.get_redirect_response_from_openid(
result=result,
@ -685,30 +691,30 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa:
)
async def _regenerate_cli_key(existing_key: str, new_key: str, user_id: Optional[str] = None) -> None:
async def _regenerate_cli_key(
existing_key: str, new_key: str, user_id: Optional[str] = None
) -> None:
"""Regenerate an existing CLI key with a new token"""
from litellm.proxy._types import RegenerateKeyRequest, UserAPIKeyAuth
from litellm.proxy.management_endpoints.key_management_endpoints import (
regenerate_key_fn,
)
verbose_proxy_logger.info(f"Regenerating existing CLI key: {existing_key}")
admin_user_dict = UserAPIKeyAuth.get_litellm_cli_user_api_key_auth()
regenerate_request = RegenerateKeyRequest(
key=existing_key,
new_key=new_key,
duration="24hr",
user_id=user_id,
)
await regenerate_key_fn(
key=existing_key,
data=regenerate_request,
user_api_key_dict=admin_user_dict
key=existing_key, data=regenerate_request, user_api_key_dict=admin_user_dict
)
verbose_proxy_logger.info(f"Regenerated CLI key: {new_key}")
@ -720,9 +726,9 @@ async def _create_new_cli_key(
from litellm.proxy.management_endpoints.key_management_endpoints import (
generate_key_helper_fn,
)
verbose_proxy_logger.info("Creating new CLI key")
await generate_key_helper_fn(
request_type="key",
duration="24hr",
@ -734,13 +740,20 @@ async def _create_new_cli_key(
table_name="key",
token=key,
)
verbose_proxy_logger.info(f"Created new CLI key: {key}")
async def cli_sso_callback(request: Request, key: Optional[str] = None, existing_key: Optional[str] = None, result: Optional[Union[OpenID, dict]] = None):
async def cli_sso_callback(
request: Request,
key: Optional[str] = None,
existing_key: Optional[str] = None,
result: Optional[Union[OpenID, dict]] = None,
):
"""CLI SSO callback - regenerates existing CLI key or creates new one"""
verbose_proxy_logger.info(f"CLI SSO callback for key: {key}, existing_key: {existing_key}")
verbose_proxy_logger.info(
f"CLI SSO callback for key: {key}, existing_key: {existing_key}"
)
from litellm.proxy.proxy_server import prisma_client
@ -754,8 +767,10 @@ async def cli_sso_callback(request: Request, key: Optional[str] = None, existing
raise HTTPException(
status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
)
parsed_openid_result = SSOAuthenticationHandler._get_user_email_and_id_from_result(result=result)
parsed_openid_result = SSOAuthenticationHandler._get_user_email_and_id_from_result(
result=result
)
verbose_proxy_logger.debug(f"parsed_openid_result: {parsed_openid_result}")
try:
@ -783,7 +798,9 @@ async def cli_sso_callback(request: Request, key: Optional[str] = None, existing
except Exception as e:
verbose_proxy_logger.error(f"Error with CLI key: {e}")
raise HTTPException(status_code=500, detail=f"Failed to process CLI key: {str(e)}")
raise HTTPException(
status_code=500, detail=f"Failed to process CLI key: {str(e)}"
)
@router.get("/sso/cli/poll/{key_id}", tags=["experimental"], include_in_schema=False)
@ -874,8 +891,10 @@ async def insert_sso_user(
auto_create_key=False,
)
if result_openid:
new_user_request.metadata = {"auth_provider": result_openid.provider}
if result_openid and hasattr(result_openid, "provider"):
new_user_request.metadata = {
"auth_provider": getattr(result_openid, "provider")
}
response = await new_user(
data=new_user_request,
@ -1052,11 +1071,13 @@ class SSOAuthenticationHandler:
# or a cryptographicly signed state that we can verify stateless
# For simplification we are using a static state, this is not perfect but some
# SSO providers do not allow stateless verification
redirect_params = SSOAuthenticationHandler._get_generic_sso_redirect_params(
state=state,
generic_authorization_endpoint=generic_authorization_endpoint
redirect_params = (
SSOAuthenticationHandler._get_generic_sso_redirect_params(
state=state,
generic_authorization_endpoint=generic_authorization_endpoint,
)
)
return await generic_sso.get_login_redirect(**redirect_params) # type: ignore
raise ValueError(
"Unknown SSO provider. Please setup SSO with client IDs https://docs.litellm.ai/docs/proxy/admin_ui_sso"
@ -1064,26 +1085,26 @@ class SSOAuthenticationHandler:
@staticmethod
def _get_generic_sso_redirect_params(
state: Optional[str] = None,
generic_authorization_endpoint: Optional[str] = None
state: Optional[str] = None,
generic_authorization_endpoint: Optional[str] = None,
) -> dict:
"""
Get redirect parameters for Generic SSO with proper state priority handling.
Priority order:
1. CLI state (if provided)
2. GENERIC_CLIENT_STATE environment variable
3. Generated UUID for Okta (if Okta endpoint detected)
Args:
state: Optional state parameter (e.g., CLI state)
generic_authorization_endpoint: Authorization endpoint URL
Returns:
dict: Redirect parameters for SSO login
"""
redirect_params = {}
if state:
# CLI state takes priority
# the litellm proxy cli sends the "state" parameter to the proxy server for auth. We should maintain the state parameter for the cli if it is provided
@ -1092,8 +1113,13 @@ class SSOAuthenticationHandler:
generic_client_state = os.getenv("GENERIC_CLIENT_STATE", None)
if generic_client_state:
redirect_params["state"] = generic_client_state
elif generic_authorization_endpoint and "okta" in generic_authorization_endpoint:
redirect_params["state"] = uuid.uuid4().hex # set state param for okta - required
elif (
generic_authorization_endpoint
and "okta" in generic_authorization_endpoint
):
redirect_params["state"] = (
uuid.uuid4().hex
) # set state param for okta - required
return redirect_params
@ -1127,11 +1153,11 @@ class SSOAuthenticationHandler:
redirect_url += sso_callback_route
else:
redirect_url += "/" + sso_callback_route
# Append existing_key as query parameter if provided
if existing_key:
redirect_url += f"?existing_key={existing_key}"
return redirect_url
@staticmethod
@ -1165,7 +1191,9 @@ class SSOAuthenticationHandler:
)
return user_info
except Exception as e:
verbose_proxy_logger.error(f"Error upserting SSO user into LiteLLM DB: {e}")
verbose_proxy_logger.exception(
f"Error upserting SSO user into LiteLLM DB: {e}"
)
return user_info
@staticmethod
@ -1314,7 +1342,9 @@ class SSOAuthenticationHandler:
return team_request
@staticmethod
def _get_cli_state(source: Optional[str], key: Optional[str], existing_key: Optional[str] = None) -> Optional[str]:
def _get_cli_state(
source: Optional[str], key: Optional[str], existing_key: Optional[str] = None
) -> Optional[str]:
"""
Checks the request 'source' if a cli state token was passed in
@ -1374,7 +1404,7 @@ class SSOAuthenticationHandler:
if user_email is not None and (user_id is None or len(user_id) == 0):
user_id = user_email
return ParsedOpenIDResult(
user_email=user_email,
user_id=user_id,
@ -1408,13 +1438,16 @@ class SSOAuthenticationHandler:
)
# User is Authe'd in - generate key for the UI to access Proxy
parsed_openid_result = SSOAuthenticationHandler._get_user_email_and_id_from_result(result=result, generic_client_id=generic_client_id)
parsed_openid_result = (
SSOAuthenticationHandler._get_user_email_and_id_from_result(
result=result, generic_client_id=generic_client_id
)
)
user_email = parsed_openid_result.get("user_email")
user_id = parsed_openid_result.get("user_id")
user_role = parsed_openid_result.get("user_role")
verbose_proxy_logger.info(f"SSO callback result: {result}")
user_info = None
user_id_models: List = []
max_internal_user_budget = litellm.max_internal_user_budget

2
litellm/proxy/management_helpers/team_member_permission_checks.py
View file

@ -171,7 +171,7 @@ class TeamMemberPermissionChecks:
"""
all_available_permissions = []
for route in LiteLLMRoutes.key_management_routes.value:
all_available_permissions.append(route.value)
all_available_permissions.append(route)
return all_available_permissions
@staticmethod

4
litellm/proxy/management_helpers/utils.py
View file

@ -1,6 +1,5 @@
# What is this?
## Helper utils for the management endpoints (keys/users/teams)
from litellm._uuid import uuid
from datetime import datetime
from functools import wraps
from typing import Optional, Tuple
@ -9,6 +8,7 @@ from fastapi import HTTPException, Request
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.proxy._types import ( # key request types; user request types; team request types; customer request types
DeleteCustomerRequest,
DeleteTeamRequest,
@ -36,7 +36,7 @@ def get_new_internal_user_defaults(
user_info = litellm.default_internal_user_params or {}
returned_dict: SSOUserDefinedValues = {
"models": user_info.get("models", None),
"models": user_info.get("models") or [],
"max_budget": user_info.get("max_budget", litellm.max_internal_user_budget),
"budget_duration": user_info.get(
"budget_duration", litellm.internal_user_budget_duration

10
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -465,14 +465,6 @@ async def anthropic_proxy_route(
region_name=None,
)
custom_headers = {}
if (
"authorization" not in request.headers
and "x-api-key" not in request.headers
and anthropic_api_key is not None
):
custom_headers["x-api-key"] = "{}".format(anthropic_api_key)
## check for streaming
is_streaming_request = await is_streaming_request_fn(request)
@ -480,7 +472,7 @@ async def anthropic_proxy_route(
endpoint_func = create_pass_through_route(
endpoint=endpoint,
target=str(updated_url),
custom_headers=custom_headers,
custom_headers={"x-api-key": "{}".format(anthropic_api_key)},
_forward_headers=True,
) # dynamically construct pass-through endpoint based on incoming path
received_value = await endpoint_func(

26
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -3,7 +3,6 @@ import asyncio
import copy
import json
import traceback
from litellm._uuid import uuid
from base64 import b64encode
from datetime import datetime
from typing import Any, Dict, List, Optional, Tuple, Union
@ -33,6 +32,7 @@ from websockets.exceptions import (
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -432,10 +432,10 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
for field_name, field_value in form_data.items():
if isinstance(field_value, (StarletteUploadFile, UploadFile)):
files[
field_name
] = await HttpPassThroughEndpointHelpers._build_request_files_from_upload_file(
upload_file=field_value
files[field_name] = (
await HttpPassThroughEndpointHelpers._build_request_files_from_upload_file(
upload_file=field_value
)
)
else:
form_data_dict[field_name] = field_value
@ -484,9 +484,11 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
user_api_key_request_route=user_api_key_dict.request_route,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_budget_reset_at=user_api_key_dict.budget_reset_at.isoformat()
if user_api_key_dict.budget_reset_at
else None,
user_api_key_budget_reset_at=(
user_api_key_dict.budget_reset_at.isoformat()
if user_api_key_dict.budget_reset_at
else None
),
)
)
@ -519,9 +521,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
"passthrough_logging_payload": passthrough_logging_payload,
}
logging_obj.model_call_details[
"passthrough_logging_payload"
] = passthrough_logging_payload
logging_obj.model_call_details["passthrough_logging_payload"] = (
passthrough_logging_payload
)
return kwargs
@ -933,7 +935,6 @@ def create_pass_through_route(
):
# check if target is an adapter.py or a url
from litellm._uuid import uuid
from litellm.proxy.types_utils.utils import get_instance_fn
try:
@ -1909,7 +1910,6 @@ async def create_pass_through_endpoints(
Create new pass-through endpoint
"""
from litellm._uuid import uuid
from litellm.proxy.proxy_server import (
get_config_general_settings,
update_config_general_settings,

6
litellm/proxy/proxy_config.yaml
View file

@ -43,4 +43,8 @@ litellm_settings:
turn_off_message_logging: true
datadog_llm_observability_params:
turn_off_message_logging: true
# proxy_config.yaml
cache: True
cache_params:
type: redis
redis_startup_nodes: [{"host": "127.0.0.1", "port": "7000"}, {"host": "127.0.0.1", "port": "7001"}, {"host": "127.0.0.1", "port": "7002"}, {"host": "127.0.0.1", "port": "7003"}]

94
litellm/proxy/proxy_server.py
View file

@ -152,6 +152,10 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
router as mcp_discoverable_endpoints_router,
)
from litellm.proxy._experimental.mcp_server.rest_endpoints import (
router as mcp_rest_endpoints_router,
)
@ -250,9 +254,7 @@ from litellm.proxy.management_endpoints.customer_endpoints import (
from litellm.proxy.management_endpoints.internal_user_endpoints import (
router as internal_user_router,
)
from litellm.proxy.management_endpoints.internal_user_endpoints import (
user_update,
)
from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
from litellm.proxy.management_endpoints.key_management_endpoints import (
delete_verification_tokens,
duration_in_seconds,
@ -299,9 +301,7 @@ from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMi
from litellm.proxy.openai_files_endpoints.files_endpoints import (
router as openai_files_router,
)
from litellm.proxy.openai_files_endpoints.files_endpoints import (
set_files_config,
)
from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_config
from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
passthrough_endpoint_router,
)
@ -644,11 +644,6 @@ async def proxy_startup_event(app: FastAPI):
user_api_key_cache=user_api_key_cache,
)
if use_background_health_checks:
asyncio.create_task(
_run_background_health_check()
) # start the background health check coroutine.
if prompt_injection_detection_obj is not None: # [TODO] - REFACTOR THIS
prompt_injection_detection_obj.update_environment(router=llm_router)
@ -671,6 +666,12 @@ async def proxy_startup_event(app: FastAPI):
await ProxyStartupEvent._update_default_team_member_budget()
# Start background health checks AFTER models are loaded and index is built
if use_background_health_checks:
asyncio.create_task(
_run_background_health_check()
) # start the background health check coroutine.
## [Optional] Initialize dd tracer
ProxyStartupEvent._init_dd_tracer()
@ -9556,5 +9557,76 @@ app.include_router(ui_discovery_endpoints_router)
########################################################
# MCP Server
########################################################
# Dynamic MCP server routes - handle /{mcp_server_name}/mcp
@app.api_route(
"/{mcp_server_name}/mcp",
methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"],
)
async def dynamic_mcp_route(mcp_server_name: str, request: Request):
"""Handle dynamic MCP server routes like /github_mcp/mcp"""
try:
# Validate that the MCP server exists
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
from litellm.types.mcp import MCPAuth
mcp_server = global_mcp_server_manager.get_mcp_server_by_name(mcp_server_name)
if mcp_server is None:
raise HTTPException(
status_code=404, detail=f"MCP server '{mcp_server_name}' not found"
)
# Create a new scope with the correct path format that the MCP handler expects
# Transform /{mcp_server_name}/mcp to /mcp/{mcp_server_name}
scope = dict(request.scope)
scope["path"] = f"/mcp/{mcp_server_name}"
# Import the MCP handler
from litellm.proxy._experimental.mcp_server.server import (
handle_streamable_http_mcp,
)
# Create a custom send function to capture the response
response_started = False
response_body = b""
response_status = 200
response_headers = []
async def custom_send(message):
nonlocal response_started, response_body, response_status, response_headers
if message["type"] == "http.response.start":
response_started = True
response_status = message["status"]
response_headers = message.get("headers", [])
elif message["type"] == "http.response.body":
response_body += message.get("body", b"")
# Call the existing MCP handler
await handle_streamable_http_mcp(
scope, receive=request.receive, send=custom_send
)
# Return the response
from starlette.responses import Response
headers_dict = {k.decode(): v.decode() for k, v in response_headers}
return Response(
content=response_body,
status_code=response_status,
headers=headers_dict,
media_type=headers_dict.get("content-type", "application/json"),
)
except Exception as e:
verbose_proxy_logger.error(
f"Error handling dynamic MCP route for {mcp_server_name}: {str(e)}"
)
raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
app.mount(path=BASE_MCP_ROUTE, app=mcp_app)
app.include_router(mcp_rest_endpoints_router)
app.include_router(mcp_discoverable_endpoints_router)

1
litellm/proxy/schema.prisma
View file

@ -225,6 +225,7 @@ model LiteLLM_VerificationToken {
auto_rotate Boolean? @default(false) // Whether this key should be auto-rotated
rotation_interval String? // How often to rotate (e.g., "30d", "90d")
last_rotation_at DateTime? // When this key was last rotated
key_rotation_at DateTime? // When this key should next be rotated
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
object_permission LiteLLM_ObjectPermissionTable? @relation(fields: [object_permission_id], references: [object_permission_id])

22
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -10,7 +10,7 @@ from pydantic import BaseModel
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import REDACTED_BY_LITELM_STRING, MAX_STRING_LENGTH_PROMPT_IN_DB
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB, REDACTED_BY_LITELM_STRING
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
@ -21,6 +21,7 @@ from litellm.types.utils import (
StandardLoggingModelInformation,
StandardLoggingPayload,
StandardLoggingVectorStoreRequest,
VectorStoreSearchResponse,
)
from litellm.utils import get_end_user_id_for_cost_tracking
@ -297,7 +298,9 @@ def get_logging_payload( # noqa: PLR0915
id = f"{id}_cache_hit{time.time()}" # SpendLogs does not allow duplicate request_id
mcp_namespaced_tool_name = None
mcp_tool_call_metadata = clean_metadata.get("mcp_tool_call_metadata", {})
mcp_tool_call_metadata: Optional[StandardLoggingMCPToolCall] = clean_metadata.get(
"mcp_tool_call_metadata"
)
if mcp_tool_call_metadata is not None:
mcp_namespaced_tool_name = mcp_tool_call_metadata.get(
"namespaced_tool_name", None
@ -505,23 +508,23 @@ def _sanitize_request_body_for_spend_logs_payload(
# This split ensures we keep more context from the end of conversations
start_ratio = 0.35
end_ratio = 0.65
# Calculate character distribution
start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * start_ratio)
end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * end_ratio)
# Ensure we don't exceed the total limit
total_keep = start_chars + end_chars
if total_keep > MAX_STRING_LENGTH_PROMPT_IN_DB:
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
# If the string length is less than what we want to keep, just truncate normally
if len(value) <= MAX_STRING_LENGTH_PROMPT_IN_DB:
return value
# Calculate how many characters are being skipped
skipped_chars = len(value) - total_keep
# Build the truncated string: beginning + truncation marker + end
truncated_value = (
f"{value[:start_chars]}"
@ -567,8 +570,9 @@ def _get_vector_store_request_for_spend_logs_payload(
if vector_store_request_metadata is None:
return None
for vector_store_request in vector_store_request_metadata:
vector_store_search_response = (
vector_store_request.get("vector_store_search_response", {}) or {}
vector_store_search_response: VectorStoreSearchResponse = (
vector_store_request.get("vector_store_search_response")
or VectorStoreSearchResponse()
)
response_data = vector_store_search_response.get("data", []) or []
for response_item in response_data:

148
litellm/router.py
View file

@ -17,7 +17,6 @@ import logging
import threading
import time
import traceback
from litellm._uuid import uuid
from collections import defaultdict
from functools import lru_cache
from typing import (
@ -45,6 +44,7 @@ import litellm.litellm_core_utils
import litellm.litellm_core_utils.exception_mapping_utils
from litellm import get_secret_str
from litellm._logging import verbose_router_logger
from litellm._uuid import uuid
from litellm.caching.caching import (
DualCache,
InMemoryCache,
@ -409,7 +409,12 @@ class Router:
) # {"TEAM_ID": PatternMatchRouter}
self.auto_routers: Dict[str, "AutoRouter"] = {}
# Initialize model ID to deployment index mapping for O(1) lookups
self.model_id_to_deployment_index_map: Dict[str, int] = {}
if model_list is not None:
# Build model index immediately to enable O(1) lookups from the start
self._build_model_id_to_deployment_index_map(model_list)
model_list = copy.deepcopy(model_list)
self.set_model_list(model_list)
self.healthy_deployments: List = self.model_list # type: ignore
@ -2005,11 +2010,17 @@ class Router:
# Filter out prompt management specific parameters from data before merging
prompt_management_params = {
"bitbucket_config", "dotprompt_config", "prompt_id",
"prompt_variables", "prompt_label", "prompt_version"
"bitbucket_config",
"dotprompt_config",
"prompt_id",
"prompt_variables",
"prompt_label",
"prompt_version",
}
filtered_data = {k: v for k, v in data.items() if k not in prompt_management_params}
filtered_data = {
k: v for k, v in data.items() if k not in prompt_management_params
}
kwargs = {**filtered_data, **kwargs, **optional_params}
kwargs["model"] = model
kwargs["messages"] = messages
@ -3436,7 +3447,7 @@ class Router:
*[try_retrieve_batch(model) for model in filtered_model_list]
)
final_results = {
final_results: Dict = {
"object": "list",
"data": [],
"first_id": None,
@ -4108,7 +4119,9 @@ class Router:
"""
model_group = kwargs.get("model")
response = original_function(*args, **kwargs)
if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
if coroutine_checker.is_async_callable(response) or inspect.isawaitable(
response
):
response = await response
## PROCESS RESPONSE HEADERS
response = await self.set_response_headers(
@ -4517,7 +4530,9 @@ class Router:
_time_to_cooldown = self.cooldown_time
if isinstance(_model_info, dict):
deployment_id = _model_info.get("id", None)
deployment_id: Optional[str] = _model_info.get("id")
if deployment_id is None:
return False
increment_deployment_failures_for_current_minute(
litellm_router_instance=self,
deployment_id=deployment_id,
@ -4974,7 +4989,7 @@ class Router:
model = deployment.to_json(exclude_none=True)
self.model_list.append(model)
self._add_model_to_list_and_index_map(model=model, model_id=deployment.model_info.id)
return deployment
except Exception as e:
if self.ignore_invalid_deployments:
@ -5085,6 +5100,7 @@ class Router:
def set_model_list(self, model_list: list):
original_model_list = copy.deepcopy(model_list)
self.model_list = []
self.model_id_to_deployment_index_map = {} # Reset the index
# we add api_base/api_key each model so load balancing between azure/gpt on api_base1 and api_base2 works
for model in original_model_list:
@ -5134,12 +5150,12 @@ class Router:
# Check if this is a prompt management model before validating as LLM provider
litellm_model = deployment.litellm_params.model
is_prompt_management_model = False
if "/" in litellm_model:
split_litellm_model = litellm_model.split("/")[0]
if split_litellm_model in litellm._known_custom_logger_compatible_callbacks:
is_prompt_management_model = True
if is_prompt_management_model:
# For prompt management models, skip LLM provider validation
# The actual model will be resolved at runtime from the prompt file
@ -5229,11 +5245,12 @@ class Router:
# litellm_router_instance=self, model=deployment.to_json(exclude_none=True)
# )
self._initialize_deployment_for_pass_through(
deployment=deployment,
custom_llm_provider=custom_llm_provider,
model=deployment.litellm_params.model,
)
if custom_llm_provider is not None:
self._initialize_deployment_for_pass_through(
deployment=deployment,
custom_llm_provider=custom_llm_provider,
model=deployment.litellm_params.model,
)
#########################################################
# Check if this is an auto-router deployment
@ -5323,10 +5340,42 @@ class Router:
self._add_deployment(deployment=deployment)
# add to model names
self.model_list.append(_deployment)
self._add_model_to_list_and_index_map(model=_deployment, model_id=deployment.model_info.id)
self.model_names.append(deployment.model_name)
return deployment
def _update_deployment_indices_after_removal(self, model_id: str, removal_idx: int) -> None:
"""
Helper method to update deployment indices after a deployment has been removed from model_list.
Parameters:
- model_id: str - the id of the deployment that was removed
- removal_idx: int - the index where the deployment was removed from model_list
"""
# Update indices for all models after the removed one
for deployment_id, idx in self.model_id_to_deployment_index_map.items():
if idx > removal_idx:
self.model_id_to_deployment_index_map[deployment_id] = idx - 1
# Remove the deleted model from index
if model_id in self.model_id_to_deployment_index_map:
del self.model_id_to_deployment_index_map[model_id]
def _add_model_to_list_and_index_map(self, model: dict, model_id: Optional[str] = None) -> None:
"""
Helper method to add a model to the model_list and update the model_id_to_deployment_index_map.
Parameters:
- model: dict - the model to add to the list
- model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"]
"""
self.model_list.append(model)
# Update model index for O(1) lookup
if model_id is not None:
self.model_id_to_deployment_index_map[model_id] = len(self.model_list) - 1
elif model.get("model_info", {}).get("id") is not None:
self.model_id_to_deployment_index_map[model["model_info"]["id"]] = len(self.model_list) - 1
def upsert_deployment(self, deployment: Deployment) -> Optional[Deployment]:
"""
Add or update deployment
@ -5352,12 +5401,15 @@ class Router:
# if there is a new litellm param -> then update the deployment
# remove the previous deployment
removal_idx: Optional[int] = None
for idx, model in enumerate(self.model_list):
if model["model_info"]["id"] == deployment.model_info.id:
removal_idx = idx
deployment_id = deployment.model_info.id
deployment_fast_mapping = self.model_id_to_deployment_index_map
if deployment_id in deployment_fast_mapping:
removal_idx = deployment_fast_mapping[deployment_id]
if removal_idx is not None:
self.model_list.pop(removal_idx)
if removal_idx is not None:
self.model_list.pop(removal_idx)
self._update_deployment_indices_after_removal(model_id=deployment_id, removal_idx=removal_idx)
# if the model_id is not in router
self.add_deployment(deployment=deployment)
@ -5381,13 +5433,14 @@ class Router:
- OR None (if deleted deployment not found)
"""
deployment_idx = None
for idx, m in enumerate(self.model_list):
if m["model_info"]["id"] == id:
deployment_idx = idx
if id in self.model_id_to_deployment_index_map:
deployment_idx = self.model_id_to_deployment_index_map[id]
try:
if deployment_idx is not None:
# Pop the item from the list first
item = self.model_list.pop(deployment_idx)
self._update_deployment_indices_after_removal(model_id=id, removal_idx=deployment_idx)
return item
else:
return None
@ -5400,15 +5453,17 @@ class Router:
Raise Exception -> if model found in invalid format
"""
for model in self.model_list:
if "model_info" in model and "id" in model["model_info"]:
if model_id == model["model_info"]["id"]:
if isinstance(model, dict):
return Deployment(**model)
elif isinstance(model, Deployment):
return model
else:
raise Exception("Model invalid format - {}".format(type(model)))
# Use O(1) lookup via model_id_to_deployment_index_map only
if model_id in self.model_id_to_deployment_index_map:
idx = self.model_id_to_deployment_index_map[model_id]
model = self.model_list[idx]
if isinstance(model, dict):
return Deployment(**model)
elif isinstance(model, Deployment):
return model
else:
raise Exception("Model invalid format - {}".format(type(model)))
return None
def get_deployment_credentials(self, model_id: str) -> Optional[dict]:
@ -6026,6 +6081,31 @@ class Router:
additional_headers[header] = value
return response
def _build_model_id_to_deployment_index_map(self, model_list: list):
"""
Build model index from model list to enable O(1) lookups immediately.
This is called during initialization to avoid the race condition where
requests arrive before model_id_to_deployment_index_map is populated.
"""
# First populate the model_list
self.model_list = []
for _, model in enumerate(model_list):
# Extract model_info from the model dict
model_info = model.get("model_info", {})
model_id = model_info.get("id")
# If no ID exists, generate one using the same logic as set_model_list
if model_id is None:
model_name = model.get("model_name", "")
litellm_params = model.get("litellm_params", {})
model_id = self._generate_model_id(model_name, litellm_params)
# Update the model_info in the original list
if "model_info" not in model:
model["model_info"] = {}
model["model_info"]["id"] = model_id
self._add_model_to_list_and_index_map(model=model, model_id=model_id)
def get_model_ids(
self, model_name: Optional[str] = None, exclude_team_models: bool = False
) -> List[str]:

9
litellm/types/llms/openai.py
View file

@ -43,9 +43,12 @@ from openai.types.responses.response import (
# Handle OpenAI SDK version compatibility for Text type
try:
from openai.types.responses.response_create_params import (
Text as ResponseText, # type: ignore
# fmt: off
from openai.types.responses.response_create_params import ( # type: ignore[attr-defined]
Text as ResponseText, # type: ignore[attr-defined]
)
# fmt: on
except (ImportError, AttributeError):
# Fall back to the concrete config type available in all SDK versions
from openai.types.responses.response_text_config_param import (
@ -1308,7 +1311,7 @@ class MCPListToolsFailedEvent(BaseLiteLLMOpenAIResponseObject):
item_id: str
# MCP Call Events
# MCP Call Events
class MCPCallInProgressEvent(BaseLiteLLMOpenAIResponseObject):
type: Literal[ResponsesAPIStreamEvents.MCP_CALL_IN_PROGRESS]
sequence_number: int

20
litellm/types/mcp.py
View file

@ -15,6 +15,7 @@ else:
MCPImageContent = Any
MCPTextContent = Any
class MCPTransport(str, enum.Enum):
sse = "sse"
http = "http"
@ -26,17 +27,21 @@ class MCPSpecVersion(str, enum.Enum):
mar_2025 = "2025-03-26"
jun_2025 = "2025-06-18"
class MCPAuth(str, enum.Enum):
none = "none"
api_key = "api_key"
bearer_token = "bearer_token"
basic = "basic"
authorization = "authorization"
oauth2 = "oauth2"
# MCP Literals
MCPTransportType = Literal[MCPTransport.sse, MCPTransport.http, MCPTransport.stdio]
MCPSpecVersionType = Literal[MCPSpecVersion.nov_2024, MCPSpecVersion.mar_2025, MCPSpecVersion.jun_2025]
MCPSpecVersionType = Literal[
MCPSpecVersion.nov_2024, MCPSpecVersion.mar_2025, MCPSpecVersion.jun_2025
]
MCPAuthType = Optional[
Literal[
MCPAuth.none,
@ -44,11 +49,11 @@ MCPAuthType = Optional[
MCPAuth.bearer_token,
MCPAuth.basic,
MCPAuth.authorization,
MCPAuth.oauth2,
]
]
class MCPServerCostInfo(TypedDict, total=False):
default_cost_per_query: Optional[float]
"""
@ -82,6 +87,7 @@ class MCPPreCallRequestObject(BaseModel):
"""
Pydantic object used for MCP pre_call_hook request validation and modification
"""
tool_name: str
arguments: Dict[str, Any]
server_name: Optional[str] = None
@ -93,6 +99,7 @@ class MCPPreCallResponseObject(BaseModel):
"""
Pydantic object used for MCP pre_call_hook response
"""
should_proceed: bool = True
modified_arguments: Optional[Dict[str, Any]] = None
error_message: Optional[str] = None
@ -103,6 +110,7 @@ class MCPDuringCallRequestObject(BaseModel):
"""
Pydantic object used for MCP during_call_hook request
"""
tool_name: str
arguments: Dict[str, Any]
server_name: Optional[str] = None
@ -114,6 +122,7 @@ class MCPDuringCallResponseObject(BaseModel):
"""
Pydantic object used for MCP during_call_hook response
"""
should_continue: bool = True
error_message: Optional[str] = None
hidden_params: HiddenParams = HiddenParams()
@ -123,5 +132,8 @@ class MCPPostCallResponseObject(BaseModel):
"""
Pydantic object used for MCP post_call_hook response
"""
mcp_tool_call_response: List[Union[MCPTextContent, MCPImageContent, MCPEmbeddedResource]]
hidden_params: HiddenParams
mcp_tool_call_response: List[
Union[MCPTextContent, MCPImageContent, MCPEmbeddedResource]
]
hidden_params: HiddenParams

Some files were not shown because too many files have changed in this diff Show more