scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

Resolve merge conflicts: integrate path validation with **kwargs approach

Browse source 
- Keep **kwargs approach (no exec()) for security
- Integrate path traversal validation from main branch
- Add URL encoding for path parameters
- Merge both test suites (edge cases + security tests)
- All 14 tests passing
This commit is contained in:
hamzaq453 2026-01-05 10:20:26 +05:00
commit 9ca7b1ad9b
304 changed files with 25834 additions and 4839 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

182
.circleci/config.yml
View file

@ -1980,6 +1980,7 @@ jobs:
- run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
- run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
- run: python ./tests/code_coverage_tests/check_fastuuid_usage.py
- run: python ./tests/code_coverage_tests/memory_test.py
- run: helm lint ./deploy/charts/litellm-helm
db_migration_disable_update_check:
@ -2008,10 +2009,13 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install apscheduler
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
name: Load Docker Database Image
command: |
docker build -t myapp . -f ./docker/Dockerfile.database
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
command: |
@ -2024,7 +2028,7 @@ jobs:
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
-v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
--name my-app \
myapp:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000
- run:
@ -2276,9 +2280,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
command: |
@ -2313,7 +2321,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -2416,9 +2424,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@ -2451,7 +2463,7 @@ jobs:
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -2502,7 +2514,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app-3 \
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
@ -2577,9 +2589,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@ -2603,7 +2619,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -2690,9 +2706,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container 1
# intentionally give bad redis credentials here
@ -2712,7 +2732,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -2733,7 +2753,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app-2 \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4001 \
--detailed_debug
@ -2826,9 +2846,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@ -2843,7 +2867,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -3058,10 +3082,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
# Run pytest and generate JUnit XML report
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
command: |
@ -3083,7 +3110,7 @@ jobs:
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -3421,6 +3448,37 @@ jobs:
--coverage.reporter=html \
--coverage.reportsDirectory=coverage/html
build_docker_database_image:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- run:
name: Upgrade Docker
command: |
curl -fsSL https://get.docker.com | sh
docker version
- run:
name: Build Docker image
command: |
docker build \
-t litellm-docker-database:ci \
-f docker/Dockerfile.database .
- run:
name: Save Docker image to workspace root
command: |
docker save litellm-docker-database:ci | gzip > litellm-docker-database.tar.gz
- persist_to_workspace:
root: .
paths:
- litellm-docker-database.tar.gz
e2e_ui_testing:
machine:
image: ubuntu-2204:2023.10.1
@ -3432,54 +3490,18 @@ jobs:
- attach_workspace:
at: ~/project
- run:
name: Upgrade Docker to v24.x (API 1.44+)
name: Load Docker Database Image
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda activate myenv
python --version
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Install Dependencies
command: |
npm install -D @playwright/test
npm install @google-cloud/vertexai
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install "openai==1.100.1"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install "mypy==1.18.2"
pip install pyarrow
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "anyio==3.7.1"
pip install "asyncio==3.4.3"
- run:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
- run:
name: Run Docker container
command: |
@ -3491,9 +3513,9 @@ jobs:
-e UI_USERNAME="admin" \
-e UI_PASSWORD="gm" \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--name my-app \
--name litellm-docker-database \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
@ -3507,7 +3529,7 @@ jobs:
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f my-app
command: docker logs -f litellm-docker-database
background: true
- run:
name: Wait for app to be ready
@ -3515,7 +3537,10 @@ jobs:
- run:
name: Run Playwright Tests
command: |
npx playwright test e2e_ui_tests/ --reporter=html --output=test-results
npx playwright test \
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
--reporter=html \
--output=test-results
no_output_timeout: 120m
- store_artifacts:
path: test-results
@ -3705,9 +3730,16 @@ workflows:
only:
- main
- /litellm_.*/
- build_docker_database_image:
filters:
branches:
only:
- main
- /litellm_.*/
- e2e_ui_testing:
requires:
- ui_build
- build_docker_database_image
filters:
branches:
only:
@ -3720,30 +3752,40 @@ workflows:
- main
- /litellm_.*/
- e2e_openai_endpoints:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_logging_guardrails_model_info_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_spend_accuracy_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_multi_instance_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_store_model_in_db_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
@ -3756,6 +3798,8 @@ workflows:
- main
- /litellm_.*/
- proxy_pass_through_endpoint_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
@ -3894,6 +3938,8 @@ workflows:
- litellm_assistants_api_testing
- auth_ui_unit_tests
- db_migration_disable_update_check:
requires:
- build_docker_database_image
filters:
branches:
only:
@ -3973,4 +4019,4 @@ workflows:
- proxy_pass_through_endpoint_tests
- check_code_and_doc_quality
- publish_proxy_extras
- guardrails_testing
- guardrails_testing

7
.gitguardian.yaml
View file

@ -84,6 +84,10 @@ secret:
- name: Langfuse test credentials in test_completion
match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
# Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
- name: Test password in e2e test fixtures
match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
# === Preventive patterns for test keys (pattern-based) ===
# Test API keys (124 instances across 45 files)
@ -102,3 +106,6 @@ secret:
- name: Test API key patterns
match: test-api-key
- name: Short fake sk keys (1–9 digits only)
match: \bsk-\d{1,9}\b

4
.gitignore vendored
View file

@ -101,3 +101,7 @@ tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
litellm/proxy/_experimental/out/guardrails/index.html
scripts/test_vertex_ai_search.py
LAZY_LOADING_IMPROVEMENTS.md
**/test-results
**/playwright-report
**/*.storageState.json
**/coverage

1
ci_cd/security_scans.sh
View file

@ -128,6 +128,7 @@ run_grype_scans() {
"GHSA-5j98-mcp5-4vw2"
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
"CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
)
# Build JSON array of allowlisted CVE IDs for jq

4
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -182,6 +182,10 @@ spec:
{{- with .Values.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- end }}

24
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -136,4 +136,26 @@ tests:
path: spec.template.spec.containers[0].volumeMounts
content:
name: litellm-config
mountPath: /etc/litellm/
mountPath: /etc/litellm/
- it: should work with lifecycle hooks
template: deployment.yaml
set:
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- echo "Container stopping"
asserts:
- exists:
path: spec.template.spec.containers[0].lifecycle
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[0]
value: /bin/sh
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[1]
value: -c
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
value: echo "Container stopping"

16
docs/my-website/docs/mcp.md
View file

@ -110,6 +110,22 @@ For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport t
<br/>
<br/>
### OAuth Configuration & Overrides
LiteLLM attempts [OAuth 2.0 Authorization Server Discovery](https://datatracker.ietf.org/doc/html/rfc8414) by default. When you create an MCP server in the UI and set `Authentication: OAuth`, LiteLLM will locate the provider metadata, dynamically register a client, and perform PKCE-based authorization without you providing any additional details.
**Customize the OAuth flow when needed:**
<Image
img={require('../img/mcp_oauth.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
- **Provide explicit client credentials** – If the MCP provider does not offer dynamic client registration or you prefer to manage the client yourself, fill in `client_id`, `client_secret`, and the desired `scopes`.
- **Override discovery URLs** – In some environments, LiteLLM might not be able to reach the provider's metadata endpoints. Use the optional `authorization_url`, `token_url`, and `registration_url` fields to point LiteLLM directly to the correct endpoints.
<br/>
### Static Headers
Sometimes your MCP server needs specific headers on every request. Maybe it's an API key, maybe it's a custom header the server expects. Instead of configuring auth, you can just set them directly.

59
docs/my-website/docs/observability/generic_api.md
View file

@ -47,6 +47,7 @@ callback_settings:
| `endpoint` | string | Yes | HTTP endpoint to send logs to |
| `headers` | dict | No | Custom headers for the request |
| `event_types` | list | No | Filter events: `llm_api_success`, `llm_api_failure`. Defaults to all events. |
| `log_format` | string | No | Output format: `json_array` (default), `ndjson`, or `single`. Controls how logs are batched and sent. |
## Pre-configured Callbacks
@ -107,4 +108,62 @@ callback_settings:
flush_interval: 60 # seconds, default: 60
```
## Log Format Options
Control how logs are formatted and sent to your endpoint.
### JSON Array (Default)
```yaml
callback_settings:
my_api:
callback_type: generic_api
endpoint: https://your-endpoint.com
log_format: json_array # default if not specified
```
Sends all logs in a batch as a single JSON array `[{log1}, {log2}, ...]`. This is the default behavior and maintains backward compatibility.
**When to use**: Most HTTP endpoints expecting batched JSON data.
### NDJSON (Newline-Delimited JSON)
```yaml
callback_settings:
my_api:
callback_type: generic_api
endpoint: https://your-endpoint.com
log_format: ndjson
```
Sends logs as newline-delimited JSON (one record per line):
```
{log1}
{log2}
{log3}
```
**When to use**: Log aggregation services like Sumo Logic, Splunk, or Datadog that support field extraction on individual records.
**Benefits**:
- Each log is ingested as a separate message
- Field Extraction Rules work at ingest time
- Better parsing and querying performance
### Single
```yaml
callback_settings:
my_api:
callback_type: generic_api
endpoint: https://your-endpoint.com
log_format: single
```
Sends each log as an individual HTTP request in parallel when the batch is flushed.
**When to use**: Endpoints that expect individual records, or when you need maximum compatibility.
**Note**: This mode sends N HTTP requests per batch (more overhead). Consider using `ndjson` instead if your endpoint supports it.

162
docs/my-website/docs/observability/levo_integration.md Normal file
View file

@ -0,0 +1,162 @@
---
sidebar_label: Levo AI
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Levo AI
<div className="levo-logo-container" style={{ marginTop: '0.5rem', marginBottom: '1rem' }}>
<div className="levo-logo-light">
<Image img={require('../../img/levo_logo.png')} />
</div>
<div className="levo-logo-dark">
<Image img={require('../../img/levo_logo_dark.png')} />
</div>
</div>
[Levo](https://levo.ai/) is an AI observability and compliance platform that provides comprehensive monitoring, analysis, and compliance tracking for LLM applications.
## Quick Start
Send all your LLM requests and responses to Levo for monitoring and analysis using LiteLLM's built-in Levo integration.
### What You'll Get
- **Complete visibility** into all LLM API calls across all providers
- **Request and response data** including prompts, completions, and metadata
- **Usage and cost tracking** with token counts and cost breakdowns
- **Error monitoring** and performance metrics
- **Compliance tracking** for audit and governance
### Setup Steps
**1. Install OpenTelemetry dependencies:**
```bash
pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc
```
**2. Enable Levo callback in your LiteLLM config:**
Add to your `litellm_config.yaml`:
```yaml
litellm_settings:
callbacks: ["levo"]
```
**3. Configure environment variables:**
[Contact Levo support](mailto:support@levo.ai) to get your collector endpoint URL, API key, organization ID, and workspace ID.
Set these required environment variables:
```bash
export LEVOAI_API_KEY="<your-levo-api-key>"
export LEVOAI_ORG_ID="<your-levo-org-id>"
export LEVOAI_WORKSPACE_ID="<your-workspace-id>"
export LEVOAI_COLLECTOR_URL="<your-levo-collector-url>"
```
**Note:** The collector URL should be the full endpoint URL provided by Levo support. It will be used exactly as provided.
**4. Start LiteLLM:**
```bash
litellm --config config.yaml
```
**5. Make requests - they'll automatically be sent to Levo!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--data '{
"model": "gpt-3.5-turbo",
"messages": [
{
"role": "user",
"content": "Hello, this is a test message"
}
]
}'
```
## What Data is Captured
| Feature | Details |
|---------|---------|
| **What is logged** | OpenTelemetry Trace Data (OTLP format) |
| **Events** | Success + Failure |
| **Format** | OTLP (OpenTelemetry Protocol) |
| **Headers** | Automatically includes `Authorization: Bearer {LEVOAI_API_KEY}`, `x-levo-organization-id`, and `x-levo-workspace-id` |
## Configuration Reference
### Required Environment Variables
| Variable | Description | Example |
|----------|-------------|---------|
| `LEVOAI_API_KEY` | Your Levo API key | `levo_abc123...` |
| `LEVOAI_ORG_ID` | Your Levo organization ID | `org-123456` |
| `LEVOAI_WORKSPACE_ID` | Your Levo workspace ID | `workspace-789` |
| `LEVOAI_COLLECTOR_URL` | Full collector endpoint URL from Levo support | `https://collector.levo.ai/v1/traces` |
### Optional Environment Variables
| Variable | Description | Default |
|----------|-------------|---------|
| `LEVOAI_ENV_NAME` | Environment name for tagging traces | `None` |
**Note:** The collector URL is used exactly as provided by Levo support. No path manipulation is performed.
## Troubleshooting
### Not seeing traces in Levo?
1. **Verify Levo callback is enabled**: Check LiteLLM startup logs for `initializing callbacks=['levo']`
2. **Check required environment variables**: Ensure all required variables are set:
```bash
echo $LEVOAI_API_KEY
echo $LEVOAI_ORG_ID
echo $LEVOAI_WORKSPACE_ID
echo $LEVOAI_COLLECTOR_URL
```
3. **Verify collector connectivity**: Test if your collector is reachable:
```bash
curl <your-collector-url>/health
```
4. **Check for initialization errors**: Look for errors in LiteLLM startup logs. Common issues:
- Missing OpenTelemetry packages: Install with `pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc`
- Missing required environment variables: All four required variables must be set
- Invalid collector URL: Ensure the URL is correct and reachable
5. **Enable debug logging**:
```bash
export LITELLM_LOG="DEBUG"
```
6. **Wait for async export**: OTLP sends traces asynchronously. Wait 10-15 seconds after making requests before checking Levo.
### Common Errors
**Error: "LEVOAI_COLLECTOR_URL environment variable is required"**
- Solution: Set the `LEVOAI_COLLECTOR_URL` environment variable with your collector endpoint URL from Levo support.
**Error: "No module named 'opentelemetry'"**
- Solution: Install OpenTelemetry packages: `pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc`
## Additional Resources
- [Levo Documentation](https://docs.levo.ai)
- [OpenTelemetry Specification](https://opentelemetry.io/docs/specs/otel/)
## Need Help?
For issues or questions about the Levo integration with LiteLLM, please [contact Levo support](mailto:support@levo.ai) or open an issue on the [LiteLLM GitHub repository](https://github.com/BerriAI/litellm/issues).

6
docs/my-website/docs/observability/opentelemetry_integration.md
View file

@ -4,7 +4,7 @@ import TabItem from '@theme/TabItem';
# OpenTelemetry - Tracing LLMs with any observability tool
OpenTelemetry is a CNCF standard for observability. It connects to any observability tool, such as Jaeger, Zipkin, Datadog, New Relic, Traceloop and others.
OpenTelemetry is a CNCF standard for observability. It connects to any observability tool, such as Jaeger, Zipkin, Datadog, New Relic, Traceloop, Levo AI and others.
<Image img={require('../../img/traceloop_dash.png')} />
@ -12,7 +12,9 @@ OpenTelemetry is a CNCF standard for observability. It connects to any observabi
From v1.81.0, the request/response will be set as attributes on the parent "Received Proxy Server Request" span by default. This allows you to see the request/response in the parent span in your observability tool.
To use the older behavior with nested "litellm_request" spans, set the following environment variable:
**Note:** When making multiple LLM calls within an external OTEL span context, the last call's attributes will overwrite previous calls' attributes on the parent span.
To use the older behavior with nested "litellm_request" spans (which creates separate spans for each call), set the following environment variable:
```shell
USE_OTEL_LITELLM_REQUEST_SPAN=true

45
docs/my-website/docs/observability/sumologic_integration.md
View file

@ -148,6 +148,51 @@ Example payload:
## Advanced Configuration
### Log Format
The Sumo Logic integration uses **NDJSON (newline-delimited JSON)** format by default. This format is optimal for Sumo Logic's parsing capabilities and allows Field Extraction Rules to work at ingest time.
#### NDJSON Format
Each log entry is sent as a separate line in the HTTP request:
```
{"id":"chatcmpl-1","model":"gpt-3.5-turbo","response_cost":0.0001,...}
{"id":"chatcmpl-2","model":"gpt-4","response_cost":0.0003,...}
{"id":"chatcmpl-3","model":"gpt-3.5-turbo","response_cost":0.0001,...}
```
#### Benefits for Field Extraction Rules (FERs)
With NDJSON format, you can create Field Extraction Rules directly:
```
_sourceCategory=litellm/logs
| json field=_raw "model", "response_cost", "user" as model, cost, user
```
**Before NDJSON** (with JSON array format):
- Required `parse regex ... multi` workaround
- FERs couldn't parse at ingest time
- Query-time parsing impacted dashboard performance
**After NDJSON**:
- ✅ FERs parse fields at ingest time
- ✅ No query-time workarounds needed
- ✅ Better dashboard performance
- ✅ Simpler query syntax
#### Changing the Log Format (Advanced)
If you need to change the log format (not recommended for Sumo Logic):
```yaml
callback_settings:
sumologic:
callback_type: generic_api
callback_name: sumologic
log_format: json_array # Override to use JSON array instead
```
### Batching Settings
Control how LiteLLM batches logs before sending to Sumo Logic:

4
docs/my-website/docs/providers/anthropic.md
View file

@ -444,7 +444,7 @@ Here's what a sample Raw Request from LiteLLM for Anthropic Context Caching look
POST Request Sent from LiteLLM:
curl -X POST \
https://api.anthropic.com/v1/messages \
-H 'accept: application/json' -H 'anthropic-version: 2023-06-01' -H 'content-type: application/json' -H 'x-api-key: sk-...' -H 'anthropic-beta: prompt-caching-2024-07-31' \
-H 'accept: application/json' -H 'anthropic-version: 2023-06-01' -H 'content-type: application/json' -H 'x-api-key: sk-...' \
-d '{'model': 'claude-3-5-sonnet-20240620', [
{
"role": "user",
@ -472,6 +472,8 @@ https://api.anthropic.com/v1/messages \
"max_tokens": 10
}'
```
**Note:** Anthropic no longer requires the `anthropic-beta: prompt-caching-2024-07-31` header. Prompt caching now works automatically when you use `cache_control` in your messages.
:::
### Caching - Large Context Caching

47
docs/my-website/docs/providers/bedrock.md
View file

@ -2208,6 +2208,53 @@ response = completion(
| `aws_role_name` | `RoleArn` | The Amazon Resource Name (ARN) of the role to assume | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
| `aws_session_name` | `RoleSessionName` | An identifier for the assumed role session | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
### IAM Roles Anywhere (On-Premise / External Workloads)
[IAM Roles Anywhere](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/introduction.html) extends IAM roles to workloads **outside of AWS** (on-premise servers, edge devices, other clouds). It uses the same STS mechanism as regular IAM roles but authenticates via X.509 certificates instead of AWS credentials.
**Setup**: Configure the [AWS Signing Helper](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/credential-helper.html) as a credential process in `~/.aws/config`:
```ini
[profile litellm-roles-anywhere]
credential_process = aws_signing_helper credential-process \
--certificate /path/to/certificate.pem \
--private-key /path/to/private-key.pem \
--trust-anchor-arn arn:aws:rolesanywhere:us-east-1:123456789012:trust-anchor/abc123 \
--profile-arn arn:aws:rolesanywhere:us-east-1:123456789012:profile/def456 \
--role-arn arn:aws:iam::123456789012:role/MyBedrockRole
```
**Usage**: Reference the profile in LiteLLM:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
aws_profile_name="litellm-roles-anywhere",
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
aws_profile_name: "litellm-roles-anywhere"
```
</TabItem>
</Tabs>
See the [IAM Roles Anywhere Getting Started Guide](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/getting-started.html) for trust anchor and profile setup.
Make the bedrock completion call

6
docs/my-website/docs/providers/bedrock_agentcore.md
View file

@ -11,6 +11,12 @@ Call Bedrock AgentCore in the OpenAI Request/Response format.
| Provider Route on LiteLLM | `bedrock/agentcore/{AGENT_RUNTIME_ARN}` |
| Provider Doc | [AWS Bedrock AgentCore ↗](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html) |
:::info
This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers, add them as you would any other MCP server. See the [MCP documentation](https://docs.litellm.ai/docs/mcp) for details.
:::
## Quick Start
### Model Format to LiteLLM

36
docs/my-website/docs/providers/zai.md
View file

@ -19,7 +19,7 @@ import os
os.environ['ZAI_API_KEY'] = ""
response = completion(
model="zai/glm-4.6",
model="zai/glm-4.7",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@ -34,7 +34,7 @@ import os
os.environ['ZAI_API_KEY'] = ""
response = completion(
model="zai/glm-4.6",
model="zai/glm-4.7",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@ -51,7 +51,8 @@ We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending complet
| Model Name | Function Call | Notes |
|------------|---------------|-------|
| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | Latest flagship model, 200K context |
| glm-4.7 | `completion(model="zai/glm-4.7", messages)` | **Latest flagship**, 200K context, **Reasoning** |
| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | 200K context |
| glm-4.5 | `completion(model="zai/glm-4.5", messages)` | 128K context |
| glm-4.5v | `completion(model="zai/glm-4.5v", messages)` | Vision model |
| glm-4.5-x | `completion(model="zai/glm-4.5-x", messages)` | Premium tier |
@ -62,16 +63,17 @@ We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending complet
## Model Pricing
| Model | Input ($/1M tokens) | Output ($/1M tokens) | Context Window |
|-------|---------------------|----------------------|----------------|
| glm-4.6 | $0.60 | $2.20 | 200K |
| glm-4.5 | $0.60 | $2.20 | 128K |
| glm-4.5v | $0.60 | $1.80 | 128K |
| glm-4.5-x | $2.20 | $8.90 | 128K |
| glm-4.5-air | $0.20 | $1.10 | 128K |
| glm-4.5-airx | $1.10 | $4.50 | 128K |
| glm-4-32b-0414-128k | $0.10 | $0.10 | 128K |
| glm-4.5-flash | **FREE** | **FREE** | 128K |
| Model | Input ($/1M tokens) | Output ($/1M tokens) | Cached Input ($/1M tokens) | Context Window |
|-------|---------------------|----------------------|---------------------------|----------------|
| glm-4.7 | $0.60 | $2.20 | $0.11 | 200K |
| glm-4.6 | $0.60 | $2.20 | - | 200K |
| glm-4.5 | $0.60 | $2.20 | - | 128K |
| glm-4.5v | $0.60 | $1.80 | - | 128K |
| glm-4.5-x | $2.20 | $8.90 | - | 128K |
| glm-4.5-air | $0.20 | $1.10 | - | 128K |
| glm-4.5-airx | $1.10 | $4.50 | - | 128K |
| glm-4-32b-0414-128k | $0.10 | $0.10 | - | 128K |
| glm-4.5-flash | **FREE** | **FREE** | - | 128K |
## Using with LiteLLM Proxy
@ -84,7 +86,7 @@ import os
os.environ['ZAI_API_KEY'] = ""
response = completion(
model="zai/glm-4.6",
model="zai/glm-4.7",
messages=[{"role": "user", "content": "Hello, how are you?"}],
)
@ -98,9 +100,9 @@ print(response.choices[0].message.content)
```yaml
model_list:
- model_name: glm-4.6
- model_name: glm-4.7
litellm_params:
model: zai/glm-4.6
model: zai/glm-4.7
api_key: os.environ/ZAI_API_KEY
- model_name: glm-4.5-flash # Free tier
litellm_params:
@ -121,7 +123,7 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "glm-4.6",
"model": "glm-4.7",
"messages": [
{
"role": "user",

4
docs/my-website/docs/proxy/config_settings.md
View file

@ -464,6 +464,9 @@ router_settings:
| DATABASE_USER | Username for database connection
| DATABASE_USERNAME | Alias for database user
| DATABRICKS_API_BASE | Base URL for Databricks API
| DATABRICKS_CLIENT_ID | Client ID for Databricks OAuth M2M authentication (Service Principal application ID)
| DATABRICKS_CLIENT_SECRET | Client secret for Databricks OAuth M2M authentication
| DATABRICKS_USER_AGENT | Custom user agent string for Databricks API requests. Used for partner telemetry attribution
| DAYS_IN_A_MONTH | Days in a month for calculation purposes. Default is 28
| DAYS_IN_A_WEEK | Days in a week for calculation purposes. Default is 7
| DAYS_IN_A_YEAR | Days in a year for calculation purposes. Default is 365
@ -708,6 +711,7 @@ router_settings:
| LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
| LITELLM_TOKEN | Access token for LiteLLM integration
| LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LOGFIRE_TOKEN | Token for Logfire logging service

2
docs/my-website/docs/proxy/configs.md
View file

@ -116,7 +116,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"role": "user",
"content": "what llm are you"
}
],
]
}
'
```

239
docs/my-website/docs/proxy/token_auth.md
View file

@ -114,6 +114,189 @@ Set `JWT_PUBLIC_KEY_URL` in your environment to a comma-separated list of URLs f
export JWT_PUBLIC_KEY_URL="https://demo.duendesoftware.com/.well-known/openid-configuration/jwks,https://accounts.google.com/.well-known/openid-configuration/jwks"
```
### Kubernetes ServiceAccount Authentication
Use Kubernetes ServiceAccount tokens to authenticate workloads running in your cluster. This is useful when you want pods to authenticate to LiteLLM using their native Kubernetes identity.
#### Prerequisites
1. Your Kubernetes cluster must have ServiceAccount token projection enabled (default in Kubernetes 1.20+)
2. Your cluster's OIDC issuer must be accessible (for EKS, GKE, AKS this is automatic)
#### Step 1: Configure the OIDC Discovery URL
Set `JWT_PUBLIC_KEY_URL` to your cluster's OIDC discovery endpoint:
<Tabs>
<TabItem value="eks" label="Amazon EKS">
```bash
# Get your EKS OIDC issuer URL
aws eks describe-cluster --name <cluster-name> --query "cluster.identity.oidc.issuer" --output text
# Set the JWKS URL (append /keys to the issuer URL)
export JWT_PUBLIC_KEY_URL="https://oidc.eks.<region>.amazonaws.com/id/<id>/keys"
```
</TabItem>
<TabItem value="gke" label="Google GKE">
```bash
# GKE uses Google's OIDC provider
export JWT_PUBLIC_KEY_URL="https://container.googleapis.com/v1/projects/<project>/locations/<location>/clusters/<cluster>/jwks"
```
</TabItem>
<TabItem value="aks" label="Azure AKS">
```bash
# Get your AKS OIDC issuer URL
az aks show --name <cluster-name> --resource-group <resource-group> --query "oidcIssuerProfile.issuerUrl" -o tsv
# Set the JWKS URL
export JWT_PUBLIC_KEY_URL="<issuer-url>/openid/v1/jwks"
```
</TabItem>
<TabItem value="self-managed" label="Self-Managed">
```bash
# For self-managed clusters, check your API server's --service-account-issuer flag
# The JWKS endpoint is typically at:
export JWT_PUBLIC_KEY_URL="https://<api-server>/openid/v1/jwks"
```
</TabItem>
</Tabs>
#### Step 2: Configure LiteLLM
Configure LiteLLM to extract identity information from Kubernetes ServiceAccount tokens:
```yaml
general_settings:
enable_jwt_auth: True
litellm_jwtauth:
# Use namespace as team identifier (resolves via team_alias in DB)
team_alias_jwt_field: "kubernetes\.io.namespace"
```
#### Step 3: Create ServiceAccount and Configure Pod
Create a ServiceAccount with an associated secret and configure your pod to use the token:
```yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: my-llm-client
namespace: my-app
---
apiVersion: v1
kind: Secret
metadata:
name: my-llm-client-token
namespace: my-app
annotations:
kubernetes.io/service-account.name: my-llm-client
type: kubernetes.io/service-account-token
---
apiVersion: v1
kind: Pod
metadata:
name: llm-client-pod
namespace: my-app
spec:
serviceAccountName: my-llm-client
containers:
- name: app
image: my-app:latest
env:
- name: LITELLM_TOKEN
valueFrom:
secretKeyRef:
name: my-llm-client-token
key: token
```
Set the expected audience in LiteLLM:
```bash
export JWT_AUDIENCE="https://kubernetes.default.svc"
```
#### Step 4: Create Team for Namespace
Create a team in LiteLLM that matches the namespace (using `team_alias`):
```bash
curl -X POST 'http://0.0.0.0:4000/team/new' \
-H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
-H 'Content-Type: application/json' \
-d '{
"team_alias": "my-app",
"team_id": "my-app",
"models": ["gpt-4", "claude-sonnet-4-20250514"]
}'
```
#### Step 5: Use the Token
From within the pod, the token is available in the `LITELLM_TOKEN` environment variable:
```bash
# Make a request to LiteLLM using the env var
curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $LITELLM_TOKEN" \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello!"}]
}'
```
#### Example: ServiceAccount Token Structure
A Kubernetes ServiceAccount token looks like this:
```json
{
"aud": ["litellm-proxy"],
"exp": 1234567890,
"iat": 1234567890,
"iss": "https://oidc.eks.us-west-2.amazonaws.com/id/EXAMPLE",
"kubernetes.io": {
"namespace": "my-app",
"pod": {
"name": "llm-client-pod",
"uid": "pod-uid"
},
"serviceaccount": {
"name": "my-llm-client",
"uid": "sa-uid"
}
},
"nbf": 1234567890,
"sub": "system:serviceaccount:my-app:my-llm-client"
}
```
#### Advanced: Map Namespace to Team Using Name Resolution
Use the `team_alias_jwt_field` to automatically resolve namespaces to teams:
```yaml
general_settings:
enable_jwt_auth: True
litellm_jwtauth:
user_id_jwt_field: "sub"
# Map the namespace to team_alias in the database
team_alias_jwt_field: "kubernetes\.io.namespace"
user_id_upsert: true
```
This way, pods in namespace `production` automatically get associated with the team that has `team_alias: production`.
### Set Accepted JWT Scope Names
Change the string in JWT 'scopes', that litellm evaluates to see if a user has admin access.
@ -183,6 +366,62 @@ litellm_jwtauth:
Now litellm will automatically update the spend for the user/team/org in the db for each call.
### Resolve by Name (Alias) Instead of ID
Sometimes your JWT token contains human-readable names instead of database IDs. LiteLLM can resolve these names to IDs by looking them up in the database.
**Use Case:** Your IDP provides team/org names in the JWT, but LiteLLM needs the actual database IDs for spend tracking and access control.
```yaml
general_settings:
master_key: sk-1234
enable_jwt_auth: True
litellm_jwtauth:
# Name-based fields (resolved via database lookup)
team_alias_jwt_field: "team_alias" # Resolves team by team_alias in DB
org_alias_jwt_field: "org_alias" # Resolves org by organization_alias in DB
```
**Expected JWT:**
```json
{
"sub": "user-123",
"team_alias": "engineering-team",
"org_alias": "acme-corp"
}
```
**How It Works:**
1. LiteLLM extracts the name from the configured JWT field
2. Looks up the entity in the database by its alias field:
- Teams: `team_alias` column in `LiteLLM_TeamTable`
- Organizations: `organization_alias` column in `LiteLLM_OrganizationTable`
3. Uses the resolved ID for spend tracking and access control
**Precedence:** ID fields always take precedence over name fields. If both `team_id_jwt_field` and `team_alias_jwt_field` are configured and both values exist in the JWT, the ID will be used.
```yaml
# Example: ID takes precedence
litellm_jwtauth:
team_id_jwt_field: "team_id" # Used if present in JWT
team_alias_jwt_field: "team_alias" # Fallback if team_id not present
```
**Nested Fields:** Name fields also support dot notation for nested claims:
```yaml
litellm_jwtauth:
team_alias_jwt_field: "organization.team.name"
org_alias_jwt_field: "company.name"
```
**Important Notes:**
- The entity (team/org) must already exist in the database with the matching alias
- Aliases should be unique - if multiple entities share the same alias, an error will be returned
- Name resolution adds a database lookup, so using IDs directly is slightly more performant
### JWT Scopes
Here's what scopes on JWT-Auth tokens look like

BIN
docs/my-website/img/levo_logo.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 12 KiB

BIN
docs/my-website/img/levo_logo_dark.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 11 KiB

146
docs/my-website/package-lock.json generated
View file

@ -8904,23 +8904,23 @@
"license": "ISC"
},
"node_modules/body-parser": {
"version": "1.20.3",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
"integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==",
"version": "1.20.4",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz",
"integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==",
"license": "MIT",
"dependencies": {
"bytes": "3.1.2",
"bytes": "~3.1.2",
"content-type": "~1.0.5",
"debug": "2.6.9",
"depd": "2.0.0",
"destroy": "1.2.0",
"http-errors": "2.0.0",
"iconv-lite": "0.4.24",
"on-finished": "2.4.1",
"qs": "6.13.0",
"raw-body": "2.5.2",
"destroy": "~1.2.0",
"http-errors": "~2.0.1",
"iconv-lite": "~0.4.24",
"on-finished": "~2.4.1",
"qs": "~6.14.0",
"raw-body": "~2.5.3",
"type-is": "~1.6.18",
"unpipe": "1.0.0"
"unpipe": "~1.0.0"
},
"engines": {
"node": ">= 0.8",
@ -8945,6 +8945,26 @@
"ms": "2.0.0"
}
},
"node_modules/body-parser/node_modules/http-errors": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
"integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
"license": "MIT",
"dependencies": {
"depd": "~2.0.0",
"inherits": "~2.0.4",
"setprototypeof": "~1.2.0",
"statuses": "~2.0.2",
"toidentifier": "~1.0.1"
},
"engines": {
"node": ">= 0.8"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/body-parser/node_modules/iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@ -8957,12 +8977,27 @@
"node": ">=0.10.0"
}
},
"node_modules/body-parser/node_modules/inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"license": "ISC"
},
"node_modules/body-parser/node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/body-parser/node_modules/statuses": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
"integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/bonjour-service": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.3.0.tgz",
@ -11873,39 +11908,39 @@
}
},
"node_modules/express": {
"version": "4.21.2",
"resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
"integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
"version": "4.22.1",
"resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz",
"integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==",
"license": "MIT",
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
"body-parser": "1.20.3",
"content-disposition": "0.5.4",
"body-parser": "~1.20.3",
"content-disposition": "~0.5.4",
"content-type": "~1.0.4",
"cookie": "0.7.1",
"cookie-signature": "1.0.6",
"cookie": "~0.7.1",
"cookie-signature": "~1.0.6",
"debug": "2.6.9",
"depd": "2.0.0",
"encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
"finalhandler": "1.3.1",
"fresh": "0.5.2",
"http-errors": "2.0.0",
"finalhandler": "~1.3.1",
"fresh": "~0.5.2",
"http-errors": "~2.0.0",
"merge-descriptors": "1.0.3",
"methods": "~1.1.2",
"on-finished": "2.4.1",
"on-finished": "~2.4.1",
"parseurl": "~1.3.3",
"path-to-regexp": "0.1.12",
"path-to-regexp": "~0.1.12",
"proxy-addr": "~2.0.7",
"qs": "6.13.0",
"qs": "~6.14.0",
"range-parser": "~1.2.1",
"safe-buffer": "5.2.1",
"send": "0.19.0",
"serve-static": "1.16.2",
"send": "~0.19.0",
"serve-static": "~1.16.2",
"setprototypeof": "1.2.0",
"statuses": "2.0.1",
"statuses": "~2.0.1",
"type-is": "~1.6.18",
"utils-merge": "1.0.1",
"vary": "~1.1.2"
@ -19281,12 +19316,12 @@
}
},
"node_modules/qs": {
"version": "6.13.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
"integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
"version": "6.14.1",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz",
"integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==",
"license": "BSD-3-Clause",
"dependencies": {
"side-channel": "^1.0.6"
"side-channel": "^1.1.0"
},
"engines": {
"node": ">=0.6"
@ -19362,15 +19397,15 @@
}
},
"node_modules/raw-body": {
"version": "2.5.2",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz",
"integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==",
"version": "2.5.3",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz",
"integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==",
"license": "MIT",
"dependencies": {
"bytes": "3.1.2",
"http-errors": "2.0.0",
"iconv-lite": "0.4.24",
"unpipe": "1.0.0"
"bytes": "~3.1.2",
"http-errors": "~2.0.1",
"iconv-lite": "~0.4.24",
"unpipe": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
@ -19385,6 +19420,26 @@
"node": ">= 0.8"
}
},
"node_modules/raw-body/node_modules/http-errors": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
"integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
"license": "MIT",
"dependencies": {
"depd": "~2.0.0",
"inherits": "~2.0.4",
"setprototypeof": "~1.2.0",
"statuses": "~2.0.2",
"toidentifier": "~1.0.1"
},
"engines": {
"node": ">= 0.8"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/raw-body/node_modules/iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@ -19397,6 +19452,21 @@
"node": ">=0.10.0"
}
},
"node_modules/raw-body/node_modules/inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"license": "ISC"
},
"node_modules/raw-body/node_modules/statuses": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
"integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/rc": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",

31
docs/my-website/src/css/custom.css
View file

@ -28,3 +28,34 @@
--ifm-color-primary-lightest: #4fddbf;
--docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3);
}
/* Levo logo sizing and theme switching */
.levo-logo-container {
position: relative;
}
.levo-logo-container img,
.levo-logo-container picture,
.levo-logo-container .ideal-image {
max-width: 200px !important;
width: 200px !important;
height: auto !important;
}
/* Show light logo by default, hide dark logo */
.levo-logo-dark {
display: none !important;
}
.levo-logo-light {
display: block !important;
}
/* In dark mode, hide light logo and show dark logo */
[data-theme='dark'] .levo-logo-light {
display: none !important;
}
[data-theme='dark'] .levo-logo-dark {
display: block !important;
}

88
docs/my-website/src/data/adopters/README.md Normal file
View file

@ -0,0 +1,88 @@
# LiteLLM Adopters
This directory contains data for organizations that use LiteLLM in production.
## Adding Your Organization
We've made it super easy to add your organization! Just follow the steps below.
### Quick Add (Recommended)
**[Edit adopters.json on GitHub →](https://github.com/BerriAI/litellm/edit/main/docs/my-website/src/data/adopters/adopters.json)**
This will open the GitHub editor in your browser where you can:
1. Add your organization's entry to the JSON array
2. Commit your changes
3. GitHub will automatically create a pull request for you!
No need to clone the repository or set up a development environment.
### JSON Format
Add your organization to the array in `adopters.json`:
```json
{
"name": "Your Organization Name",
"logoUrl": "https://yoursite.com/logo.svg",
"url": "https://yourcompany.com",
"description": "Brief description of how you use LiteLLM (shown on hover)"
}
```
### Fields
- **`name`** (required): Your organization's display name
- **`logoUrl`** (required): URL to your logo - can be either:
- External URL: `https://yoursite.com/logo.svg` (easiest!)
- Local path: `/img/adopters/your-logo.svg` (requires uploading logo file)
- **`url`** (optional): Your organization's website (makes the logo clickable)
- **`description`** (optional): Brief description shown when users hover over your logo
### Logo Options
#### Option 1: External URL (Easiest)
Simply provide a direct link to your logo hosted anywhere:
```json
"logoUrl": "https://yourcompany.com/assets/logo.svg"
```
#### Option 2: Local Logo (Better Performance)
If you prefer to host the logo locally:
1. Add your logo to `docs/my-website/static/img/adopters/your-company.svg`
2. Reference it as: `"logoUrl": "/img/adopters/your-company.svg"`
**Logo Specifications:**
- **Format**: SVG preferred (PNG also acceptable)
- **Dimensions**: 240x160px or similar 3:2 ratio recommended
- **Background**: Transparent or white background works best
### Example
```json
{
"name": "Acme Corporation",
"logoUrl": "https://acme.com/logo.svg",
"url": "https://acme.com",
"description": "Using LiteLLM to route requests across 50+ LLM providers"
}
```
### Display Order
Adopters are displayed alphabetically by organization name, so your position will be determined automatically.
### Need Help?
If you have questions about adding your organization:
- Ask in [GitHub Discussions](https://github.com/BerriAI/litellm/discussions)
- Join our [Discord community](https://discord.com/invite/wuPM9dRgDw)
Thank you for supporting LiteLLM! 🚅

8
docs/my-website/src/data/adopters/adopters.json Normal file
View file

@ -0,0 +1,8 @@
[
{
"name": "Your Logo Here",
"logoUrl": "/img/adopters/placeholder-company.svg",
"description": "Add your organization to show support for LiteLLM",
"url": "https://github.com/BerriAI/litellm/edit/main/docs/my-website/src/data/adopters/adopters.json"
}
]

23
docs/my-website/src/data/adopters/index.js Normal file
View file

@ -0,0 +1,23 @@
import adoptersData from './adopters.json';
/**
* @typedef {Object} Adopter
* @property {string} name - The organization's display name
* @property {string} logoUrl - URL to the organization's logo
* @property {string} [url] - The organization's website URL
* @property {string} [description] - Brief description shown on hover
*/
/**
* List of organizations using LiteLLM
* @type {Adopter[]}
*/
export const adopters = adoptersData;
/**
* Adopters sorted alphabetically by name
* @type {Adopter[]}
*/
export const sortedAdopters = [...adopters].sort((a, b) =>
a.name.localeCompare(b.name)
);

8
docs/my-website/static/img/adopters/placeholder-company.svg Normal file
View file

@ -0,0 +1,8 @@
<svg width="240" height="160" viewBox="0 0 240 160" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect width="240" height="160" rx="8" fill="#f8fafc"/>
<rect x="1" y="1" width="238" height="158" rx="7" stroke="#e2e8f0" stroke-width="2" stroke-dasharray="8 4"/>
<circle cx="120" cy="60" r="24" fill="#e2e8f0"/>
<path d="M120 48v24M108 60h24" stroke="#94a3b8" stroke-width="3" stroke-linecap="round"/>
<text x="120" y="110" text-anchor="middle" fill="#64748b" font-family="system-ui, -apple-system, sans-serif" font-size="14" font-weight="500">Add Your Logo</text>
<text x="120" y="130" text-anchor="middle" fill="#94a3b8" font-family="system-ui, -apple-system, sans-serif" font-size="11">Click to contribute</text>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 736 B

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260102131258_add_metadata_urls_to_mcp_servers/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "authorization_url" TEXT,
ADD COLUMN "registration_url" TEXT,
ADD COLUMN "token_url" TEXT;

3
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -208,6 +208,9 @@ model LiteLLM_MCPServerTable {
command String?
args String[] @default([])
env Json? @default("{}")
authorization_url String?
token_url String?
registration_url String?
}
// Generate Tokens for Proxy

25
litellm/__init__.py
View file

@ -136,6 +136,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
"gitlab",
"cloudzero",
"posthog",
"levo",
]
cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
@ -553,6 +554,8 @@ docker_model_runner_models: Set = set()
amazon_nova_models: Set = set()
stability_models: Set = set()
github_copilot_models: Set = set()
minimax_models: Set = set()
aws_polly_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
@ -801,6 +804,10 @@ def add_known_models():
stability_models.add(key)
elif value.get("litellm_provider") == "github_copilot":
github_copilot_models.add(key)
elif value.get("litellm_provider") == "minimax":
minimax_models.add(key)
elif value.get("litellm_provider") == "aws_polly":
aws_polly_models.add(key)
add_known_models()
@ -1005,6 +1012,8 @@ models_by_provider: dict = {
"amazon_nova": amazon_nova_models,
"stability": stability_models,
"github_copilot": github_copilot_models,
"minimax": minimax_models,
"aws_polly": aws_polly_models,
}
# mapping for those models which have larger equivalents
@ -1049,8 +1058,8 @@ openai_image_generation_models = ["dall-e-2", "dall-e-3"]
openai_video_generation_models = ["sora-2"]
# timeout is lazy-loaded via __getattr__
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.core_helpers import remove_index_from_tool_calls
# get_llm_provider is lazy-loaded via __getattr__
# remove_index_from_tool_calls is lazy-loaded via __getattr__
# Import KeyManagementSettings here (before utils import) because _key_management_settings
# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils)
@ -1499,6 +1508,7 @@ if TYPE_CHECKING:
get_first_chars_messages: Callable[..., str]
get_provider_fields: Callable[..., List]
get_valid_models: Callable[..., list]
remove_index_from_tool_calls: Callable[..., None]
# Response types - truly lazy loaded only (not in main.py or elsewhere)
ModelResponseListIterator: Type[Any]
@ -1650,6 +1660,17 @@ def __getattr__(name: str) -> Any:
LoggingCallbackManager = __getattr__("LoggingCallbackManager")
_globals["logging_callback_manager"] = LoggingCallbackManager()
return _globals["logging_callback_manager"]
# Lazy load _service_logger module
if name == "_service_logger":
from ._lazy_imports import _get_litellm_globals
_globals = _get_litellm_globals()
# Check if already cached
if "_service_logger" not in _globals:
# Import the module lazily
import litellm._service_logger
_globals["_service_logger"] = litellm._service_logger
return _globals["_service_logger"]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

9
litellm/_lazy_imports.py
View file

@ -34,6 +34,7 @@ from ._lazy_imports_registry import (
DOTPROMPT_NAMES,
LLM_CONFIG_NAMES,
TYPES_NAMES,
LLM_PROVIDER_LOGIC_NAMES,
# Import maps
_UTILS_IMPORT_MAP,
_COST_CALCULATOR_IMPORT_MAP,
@ -45,6 +46,7 @@ from ._lazy_imports_registry import (
_DOTPROMPT_IMPORT_MAP,
_TYPES_IMPORT_MAP,
_LLM_CONFIGS_IMPORT_MAP,
_LLM_PROVIDER_LOGIC_IMPORT_MAP,
)
@ -181,6 +183,8 @@ def _get_lazy_import_registry() -> dict[str, Callable[[str], Any]]:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_configs
for name in TYPES_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_types
for name in LLM_PROVIDER_LOGIC_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_provider_logic
return _LAZY_IMPORT_REGISTRY
@ -297,6 +301,11 @@ def _lazy_import_litellm_logging(name: str) -> Any:
"""Handler for litellm_logging module (Logging, modify_integration)"""
return _generic_lazy_import(name, _LITELLM_LOGGING_IMPORT_MAP, "Litellm logging")
def _lazy_import_llm_provider_logic(name: str) -> Any:
"""Handler for LLM provider logic functions (get_llm_provider, etc.)"""
return _generic_lazy_import(name, _LLM_PROVIDER_LOGIC_IMPORT_MAP, "LLM provider logic")
# ============================================================================
# SPECIAL HANDLERS
# ============================================================================

16
litellm/_lazy_imports_registry.py
View file

@ -32,6 +32,7 @@ UTILS_NAMES = (
"ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse",
"TranscriptionResponse", "TextCompletionResponse", "get_provider_fields",
"ModelResponseListIterator", "get_valid_models", "timeout",
"get_llm_provider", "remove_index_from_tool_calls",
)
# Token counter names that support lazy loading via _lazy_import_token_counter
@ -287,6 +288,12 @@ TYPES_NAMES = (
# is accessed during import time in secret_managers/main.py
)
# LLM provider logic names that support lazy loading via _lazy_import_llm_provider_logic
LLM_PROVIDER_LOGIC_NAMES = (
"get_llm_provider",
"remove_index_from_tool_calls",
)
# Import maps for registry pattern - reduces repetition
_UTILS_IMPORT_MAP = {
"exception_type": (".utils", "exception_type"),
@ -330,6 +337,8 @@ _UTILS_IMPORT_MAP = {
"ModelResponseListIterator": (".utils", "ModelResponseListIterator"),
"get_valid_models": (".utils", "get_valid_models"),
"timeout": (".timeout", "timeout"),
"get_llm_provider": ("litellm.litellm_core_utils.get_llm_provider_logic", "get_llm_provider"),
"remove_index_from_tool_calls": ("litellm.litellm_core_utils.core_helpers", "remove_index_from_tool_calls"),
}
_COST_CALCULATOR_IMPORT_MAP = {
@ -386,6 +395,11 @@ _TYPES_IMPORT_MAP = {
"LoggingCallbackManager": ("litellm.litellm_core_utils.logging_callback_manager", "LoggingCallbackManager"),
}
_LLM_PROVIDER_LOGIC_IMPORT_MAP = {
"get_llm_provider": ("litellm.litellm_core_utils.get_llm_provider_logic", "get_llm_provider"),
"remove_index_from_tool_calls": ("litellm.litellm_core_utils.core_helpers", "remove_index_from_tool_calls"),
}
_LLM_CONFIGS_IMPORT_MAP = {
"AmazonConverseConfig": (".llms.bedrock.chat.converse_transformation", "AmazonConverseConfig"),
"OpenAILikeChatConfig": (".llms.openai_like.chat.handler", "OpenAILikeChatConfig"),
@ -587,6 +601,7 @@ __all__ = [
"DOTPROMPT_NAMES",
"LLM_CONFIG_NAMES",
"TYPES_NAMES",
"LLM_PROVIDER_LOGIC_NAMES",
# Import maps
"_UTILS_IMPORT_MAP",
"_COST_CALCULATOR_IMPORT_MAP",
@ -598,5 +613,6 @@ __all__ = [
"_DOTPROMPT_IMPORT_MAP",
"_TYPES_IMPORT_MAP",
"_LLM_CONFIGS_IMPORT_MAP",
"_LLM_PROVIDER_LOGIC_IMPORT_MAP",
]

62
litellm/google_genai/adapters/transformation.py
View file

@ -8,8 +8,10 @@ from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionAssistantToolCall,
ChatCompletionImageObject,
ChatCompletionRequest,
ChatCompletionSystemMessage,
ChatCompletionTextObject,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolChoiceValues,
ChatCompletionToolMessage,
@ -385,13 +387,36 @@ class GoogleGenAIAdapter:
if role == "user":
# Handle user messages with potential function responses
combined_text = ""
content_parts: List[
Union[ChatCompletionTextObject, ChatCompletionImageObject]
] = []
tool_messages: List[ChatCompletionToolMessage] = []
for part in parts:
if isinstance(part, dict):
if "text" in part:
combined_text += part["text"]
content_parts.append(
cast(
ChatCompletionTextObject,
{"type": "text", "text": part["text"]},
)
)
elif "inline_data" in part:
# Handle Base64 image data
inline_data = part["inline_data"]
mime_type = inline_data.get("mime_type", "image/jpeg")
data = inline_data.get("data", "")
content_parts.append(
cast(
ChatCompletionImageObject,
{
"type": "image_url",
"image_url": {
"url": f"data:{mime_type};base64,{data}"
},
},
)
)
elif "functionResponse" in part:
# Transform function response to tool message
func_response = part["functionResponse"]
@ -402,13 +427,33 @@ class GoogleGenAIAdapter:
)
tool_messages.append(tool_message)
elif isinstance(part, str):
combined_text += part
content_parts.append(
cast(
ChatCompletionTextObject, {"type": "text", "text": part}
)
)
# Add user message if there's text content
if combined_text:
messages.append(
ChatCompletionUserMessage(role="user", content=combined_text)
)
# Add user message if there's content
if content_parts:
# If only one text part, use simple string format for backward compatibility
if (
len(content_parts) == 1
and isinstance(content_parts[0], dict)
and content_parts[0].get("type") == "text"
):
text_part = cast(ChatCompletionTextObject, content_parts[0])
messages.append(
ChatCompletionUserMessage(
role="user", content=text_part["text"]
)
)
else:
# Use multimodal format (array of content parts)
messages.append(
ChatCompletionUserMessage(
role="user", content=content_parts
)
)
# Add tool messages
messages.extend(tool_messages)
@ -468,7 +513,6 @@ class GoogleGenAIAdapter:
Dict in Google GenAI generate_content response format
"""
# Extract the main response content
choice = response.choices[0] if response.choices else None
if not choice:

47
litellm/integrations/custom_guardrail.py
View file

@ -243,14 +243,14 @@ class CustomGuardrail(CustomLogger):
def _is_valid_response_type(self, result: Any) -> bool:
"""
Check if result is a valid LLMResponseTypes instance.
Safely handles TypedDict types which don't support isinstance checks.
For non-LiteLLM responses (like passthrough httpx.Response), returns True
to allow them through.
"""
if result is None:
return False
try:
# Try isinstance check on valid types that support it
response_types = get_args(LLMResponseTypes)
@ -506,6 +506,7 @@ class CustomGuardrail(CustomLogger):
duration: Optional[float] = None,
masked_entity_count: Optional[Dict[str, int]] = None,
guardrail_provider: Optional[str] = None,
event_type: Optional[GuardrailEventHooks] = None,
) -> None:
"""
Builds `StandardLoggingGuardrailInformation` and adds it to the request metadata so it can be used for logging to DataDog, Langfuse, etc.
@ -514,14 +515,19 @@ class CustomGuardrail(CustomLogger):
guardrail_json_response = str(guardrail_json_response)
from litellm.types.utils import GuardrailMode
# Use event_type if provided, otherwise fall back to self.event_hook
guardrail_mode: Union[GuardrailEventHooks, GuardrailMode, List[GuardrailEventHooks]]
if event_type is not None:
guardrail_mode = event_type
elif isinstance(self.event_hook, Mode):
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump())) # type: ignore[typeddict-item]
else:
guardrail_mode = self.event_hook # type: ignore[assignment]
slg = StandardLoggingGuardrailInformation(
guardrail_name=self.guardrail_name,
guardrail_provider=guardrail_provider,
guardrail_mode=(
GuardrailMode(**self.event_hook.model_dump()) # type: ignore
if isinstance(self.event_hook, Mode)
else self.event_hook
),
guardrail_mode=guardrail_mode,
guardrail_response=guardrail_json_response,
guardrail_status=guardrail_status,
start_time=start_time,
@ -589,6 +595,7 @@ class CustomGuardrail(CustomLogger):
start_time: Optional[float] = None,
end_time: Optional[float] = None,
duration: Optional[float] = None,
event_type: Optional[GuardrailEventHooks] = None,
):
"""
Add StandardLoggingGuardrailInformation to the request data
@ -605,6 +612,7 @@ class CustomGuardrail(CustomLogger):
duration=duration,
start_time=start_time,
end_time=end_time,
event_type=event_type,
)
return response
@ -615,6 +623,7 @@ class CustomGuardrail(CustomLogger):
start_time: Optional[float] = None,
end_time: Optional[float] = None,
duration: Optional[float] = None,
event_type: Optional[GuardrailEventHooks] = None,
):
"""
Add StandardLoggingGuardrailInformation to the request data
@ -628,6 +637,7 @@ class CustomGuardrail(CustomLogger):
duration=duration,
start_time=start_time,
end_time=end_time,
event_type=event_type,
)
raise e
@ -712,16 +722,32 @@ def log_guardrail_information(func):
Logs for:
- pre_call
- during_call
- TODO: log post_call. This is more involved since the logs are sent to DD, s3 before the guardrail is even run
- post_call
"""
import asyncio
import functools
def _infer_event_type_from_function_name(
func_name: str,
) -> Optional[GuardrailEventHooks]:
"""Infer the actual event type from the function name"""
if func_name == "async_pre_call_hook":
return GuardrailEventHooks.pre_call
elif func_name == "async_moderation_hook":
return GuardrailEventHooks.during_call
elif func_name in (
"async_post_call_success_hook",
"async_post_call_streaming_hook",
):
return GuardrailEventHooks.post_call
return None
@functools.wraps(func)
async def async_wrapper(*args, **kwargs):
start_time = datetime.now() # Move start_time inside the wrapper
self: CustomGuardrail = args[0]
request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {}
event_type = _infer_event_type_from_function_name(func.__name__)
try:
response = await func(*args, **kwargs)
return self._process_response(
@ -730,6 +756,7 @@ def log_guardrail_information(func):
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
except Exception as e:
return self._process_error(
@ -738,6 +765,7 @@ def log_guardrail_information(func):
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
@functools.wraps(func)
@ -745,18 +773,21 @@ def log_guardrail_information(func):
start_time = datetime.now() # Move start_time inside the wrapper
self: CustomGuardrail = args[0]
request_data: dict = kwargs.get("data") or kwargs.get("request_data") or {}
event_type = _infer_event_type_from_function_name(func.__name__)
try:
response = func(*args, **kwargs)
return self._process_response(
response=response,
request_data=request_data,
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
except Exception as e:
return self._process_error(
e=e,
request_data=request_data,
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
@functools.wraps(func)

76
litellm/integrations/generic_api/generic_api_callback.py
View file

@ -25,6 +25,7 @@ from litellm.llms.custom_httpx.http_handler import (
from litellm.types.utils import StandardLoggingPayload
API_EVENT_TYPES = Literal["llm_api_success", "llm_api_failure"]
LOG_FORMAT_TYPES = Literal["json_array", "ndjson", "single"]
def load_compatible_callbacks() -> Dict:
@ -101,6 +102,7 @@ class GenericAPILogger(CustomBatchLogger):
headers: Optional[dict] = None,
event_types: Optional[List[API_EVENT_TYPES]] = None,
callback_name: Optional[str] = None,
log_format: Optional[LOG_FORMAT_TYPES] = None,
**kwargs,
):
"""
@ -111,6 +113,7 @@ class GenericAPILogger(CustomBatchLogger):
headers: Optional[dict] = None,
event_types: Optional[List[API_EVENT_TYPES]] = None,
callback_name: Optional[str] = None - If provided, loads config from generic_api_compatible_callbacks.json
log_format: Optional[LOG_FORMAT_TYPES] = None - Format for log output: "json_array" (default), "ndjson", or "single"
"""
#########################################################
# Check if callback_name is provided and load config
@ -135,6 +138,9 @@ class GenericAPILogger(CustomBatchLogger):
if event_types is None and "event_types" in callback_config:
event_types = callback_config["event_types"]
if log_format is None and "log_format" in callback_config:
log_format = callback_config["log_format"]
else:
verbose_logger.warning(
f"callback_name '{callback_name}' not found in generic_api_compatible_callbacks.json"
@ -156,8 +162,16 @@ class GenericAPILogger(CustomBatchLogger):
self.endpoint: str = endpoint
self.event_types: Optional[List[API_EVENT_TYPES]] = event_types
self.callback_name: Optional[str] = callback_name
# Validate and store log_format
if log_format is not None and log_format not in ["json_array", "ndjson", "single"]:
raise ValueError(
f"Invalid log_format: {log_format}. Must be one of: 'json_array', 'ndjson', 'single'"
)
self.log_format: LOG_FORMAT_TYPES = log_format or "json_array"
verbose_logger.debug(
f"in init GenericAPILogger, callback_name: {self.callback_name}, endpoint {self.endpoint}, headers {self.headers}, event_types: {self.event_types}"
f"in init GenericAPILogger, callback_name: {self.callback_name}, endpoint {self.endpoint}, headers {self.headers}, event_types: {self.event_types}, log_format: {self.log_format}"
)
#########################################################
@ -289,25 +303,65 @@ class GenericAPILogger(CustomBatchLogger):
async def async_send_batch(self):
"""
Sends the batch of messages to Generic API Endpoint
Supports three formats:
- json_array: Sends all logs as a JSON array (default)
- ndjson: Sends logs as newline-delimited JSON
- single: Sends each log as individual HTTP request in parallel
"""
try:
if not self.log_queue:
return
verbose_logger.debug(
f"Generic API Logger - about to flush {len(self.log_queue)} events"
f"Generic API Logger - about to flush {len(self.log_queue)} events in '{self.log_format}' format"
)
# make POST request to Generic API Endpoint
response = await self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=safe_dumps(self.log_queue),
)
if self.log_format == "single":
# Send each log as individual HTTP request in parallel
tasks = []
for log_entry in self.log_queue:
task = self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=safe_dumps(log_entry),
)
tasks.append(task)
verbose_logger.debug(
f"Generic API Logger - sent batch to {self.endpoint}, status code {response.status_code}"
)
# Execute all requests in parallel
responses = await asyncio.gather(*tasks, return_exceptions=True)
# Log results
for idx, result in enumerate(responses):
if isinstance(result, Exception):
verbose_logger.exception(
f"Generic API Logger - Error sending log {idx}: {result}"
)
else:
# result is a Response object
verbose_logger.debug(
f"Generic API Logger - sent log {idx}, status: {result.status_code}" # type: ignore
)
else:
# Format the payload based on log_format
if self.log_format == "json_array":
data = safe_dumps(self.log_queue)
elif self.log_format == "ndjson":
data = "\n".join(safe_dumps(log) for log in self.log_queue)
else:
raise ValueError(f"Unknown log_format: {self.log_format}")
# Make POST request
response = await self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=data,
)
verbose_logger.debug(
f"Generic API Logger - sent batch to {self.endpoint}, "
f"status: {response.status_code}, format: {self.log_format}"
)
except Exception as e:
verbose_logger.exception(

3
litellm/integrations/generic_api/generic_api_compatible_callbacks.json
View file

@ -22,6 +22,7 @@
"headers": {
"Content-Type": "application/json"
},
"environment_variables": ["SUMOLOGIC_WEBHOOK_URL"]
"environment_variables": ["SUMOLOGIC_WEBHOOK_URL"],
"log_format": "ndjson"
}
}

43
litellm/integrations/langfuse/langfuse.py
View file

@ -3,14 +3,27 @@
import os
import traceback
from datetime import datetime
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union, cast
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
List,
Optional,
Tuple,
Union,
cast,
)
from packaging.version import Version
import litellm
from litellm._logging import verbose_logger
from litellm.constants import MAX_LANGFUSE_INITIALIZED_CLIENTS
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
from litellm.litellm_core_utils.core_helpers import (
safe_deep_copy,
reconstruct_model_name,
)
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.secret_managers.main import str_to_bool
@ -437,12 +450,17 @@ class LangFuseLogger:
)
)
custom_llm_provider = cast(Optional[str], kwargs.get("custom_llm_provider"))
model_name = reconstruct_model_name(
kwargs.get("model", ""), custom_llm_provider, metadata
)
trace.generation(
CreateGeneration(
name=metadata.get("generation_name", "litellm-completion"),
startTime=start_time,
endTime=end_time,
model=kwargs["model"],
model=model_name,
modelParameters=optional_params,
prompt=input,
completion=output,
@ -543,7 +561,9 @@ class LangFuseLogger:
# as we want to fall back to litellm_call_id instead for better traceability.
# Note: Users can still explicitly set a UUID trace_id via metadata["trace_id"] (highest priority)
if trace_id is None and standard_logging_object is not None:
standard_trace_id = cast(Optional[str], standard_logging_object.get("trace_id"))
standard_trace_id = cast(
Optional[str], standard_logging_object.get("trace_id")
)
# Only use standard_logging_object.trace_id if it's not a UUID
# UUIDs are 36 characters with hyphens in format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
# We check for this specific pattern to avoid rejecting valid trace_ids that happen to have hyphens
@ -575,7 +595,9 @@ class LangFuseLogger:
mask_output = clean_metadata.pop("mask_output", False)
# Look for masking function in the dedicated location first (set by scrub_sensitive_keys_in_metadata)
# Fall back to metadata for backwards compatibility
masking_function = litellm_params.get("_langfuse_masking_function") or clean_metadata.pop("langfuse_masking_function", None)
masking_function = litellm_params.get(
"_langfuse_masking_function"
) or clean_metadata.pop("langfuse_masking_function", None)
# Apply custom masking function if provided
if masking_function is not None and callable(masking_function):
@ -776,12 +798,17 @@ class LangFuseLogger:
if system_fingerprint is not None:
optional_params["system_fingerprint"] = system_fingerprint
custom_llm_provider = cast(Optional[str], kwargs.get("custom_llm_provider"))
model_name = reconstruct_model_name(
kwargs.get("model", ""), custom_llm_provider, metadata
)
generation_params = {
"name": generation_name,
"id": clean_metadata.pop("generation_id", generation_id),
"start_time": start_time,
"end_time": end_time,
"model": kwargs["model"],
"model": model_name,
"model_parameters": optional_params,
"input": input if not mask_input else "redacted-by-litellm",
"output": output if not mask_output else "redacted-by-litellm",
@ -918,7 +945,9 @@ class LangFuseLogger:
return Version(self.langfuse_sdk_version) >= Version("2.7.3")
@staticmethod
def _apply_masking_function(data: Any, masking_function: Callable[[Any], Any]) -> Any:
def _apply_masking_function(
data: Any, masking_function: Callable[[Any], Any]
) -> Any:
"""
Apply a masking function to data, handling different data types.

125
litellm/integrations/levo/README.md Normal file
View file

@ -0,0 +1,125 @@
# Levo AI Integration
This integration enables sending LLM observability data to Levo AI using OpenTelemetry (OTLP) protocol.
## Overview
The Levo integration extends LiteLLM's OpenTelemetry support to automatically send traces to Levo's collector endpoint with proper authentication and routing headers.
## Features
- **Automatic OTLP Export**: Sends OpenTelemetry traces to Levo collector
- **Levo-Specific Headers**: Automatically includes `x-levo-organization-id` and `x-levo-workspace-id` for routing
- **Simple Configuration**: Just use `callbacks: ["levo"]` in your LiteLLM config
- **Environment-Based Setup**: Configure via environment variables
## Quick Start
### 1. Install Dependencies
```bash
pip install opentelemetry-api opentelemetry-sdk opentelemetry-exporter-otlp-proto-http opentelemetry-exporter-otlp-proto-grpc
```
### 2. Configure LiteLLM
Add to your `litellm_config.yaml`:
```yaml
litellm_settings:
callbacks: ["levo"]
```
### 3. Set Environment Variables
```bash
export LEVOAI_API_KEY="<your-levo-api-key>"
export LEVOAI_ORG_ID="<your-levo-org-id>"
export LEVOAI_WORKSPACE_ID="<your-workspace-id>"
export LEVOAI_COLLECTOR_URL="<your-levo-collector-url>"
```
### 4. Start LiteLLM
```bash
litellm --config config.yaml
```
All LLM requests will now automatically be sent to Levo!
## Configuration
### Required Environment Variables
| Variable | Description |
|----------|-------------|
| `LEVOAI_API_KEY` | Your Levo API key for authentication |
| `LEVOAI_ORG_ID` | Your Levo organization ID for routing |
| `LEVOAI_WORKSPACE_ID` | Your Levo workspace ID for routing |
| `LEVOAI_COLLECTOR_URL` | Full collector endpoint URL from Levo support |
### Optional Environment Variables
| Variable | Description | Default |
|----------|-------------|---------|
| `LEVOAI_ENV_NAME` | Environment name for tagging traces | `None` |
**Important**: The `LEVOAI_COLLECTOR_URL` is used exactly as provided. No path manipulation is performed.
## How It Works
1. **LevoLogger** extends LiteLLM's `OpenTelemetry` class
2. **Configuration** is read from environment variables via `get_levo_config()`
3. **OTLP Headers** are automatically set:
- `Authorization: Bearer {LEVOAI_API_KEY}`
- `x-levo-organization-id: {LEVOAI_ORG_ID}`
- `x-levo-workspace-id: {LEVOAI_WORKSPACE_ID}`
4. **Traces** are sent to the collector endpoint in OTLP format
## Code Structure
```
litellm/integrations/levo/
├── __init__.py # Exports LevoLogger
├── levo.py # LevoLogger implementation
└── README.md # This file
```
### Key Classes
- **LevoLogger**: Extends `OpenTelemetry`, handles Levo-specific configuration
- **LevoConfig**: Pydantic model for Levo configuration (defined in `levo.py`)
## Testing
See the test files in `tests/test_litellm/integrations/levo/`:
- `test_levo.py`: Unit tests for configuration
- `test_levo_integration.py`: Integration tests for callback registration
## Error Handling
The integration validates all required environment variables at initialization:
- Missing `LEVOAI_API_KEY`: Raises `ValueError` with clear message
- Missing `LEVOAI_ORG_ID`: Raises `ValueError` with clear message
- Missing `LEVOAI_WORKSPACE_ID`: Raises `ValueError` with clear message
- Missing `LEVOAI_COLLECTOR_URL`: Raises `ValueError` with clear message
## Integration with LiteLLM
The Levo callback is registered in:
- `litellm/litellm_core_utils/custom_logger_registry.py`: Maps `"levo"` to `LevoLogger`
- `litellm/litellm_core_utils/litellm_logging.py`: Instantiates `LevoLogger` when `callbacks: ["levo"]` is used
- `litellm/__init__.py`: Added to `_custom_logger_compatible_callbacks_literal`
## Documentation
For detailed documentation, see:
- [LiteLLM Levo Integration Docs](../../../../docs/my-website/docs/observability/levo_integration.md)
- [Levo Documentation](https://docs.levo.ai)
## Support
For issues or questions:
- LiteLLM Issues: https://github.com/BerriAI/litellm/issues
- Levo Support: support@levo.ai

3
litellm/integrations/levo/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from litellm.integrations.levo.levo import LevoLogger
__all__ = ["LevoLogger"]

117
litellm/integrations/levo/levo.py Normal file
View file

@ -0,0 +1,117 @@
import os
from typing import TYPE_CHECKING, Any, Optional, Union
from litellm.integrations.opentelemetry import OpenTelemetry
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.integrations.opentelemetry import OpenTelemetryConfig as _OpenTelemetryConfig
from litellm.types.integrations.arize import Protocol as _Protocol
Protocol = _Protocol
OpenTelemetryConfig = _OpenTelemetryConfig
Span = Union[_Span, Any]
else:
Protocol = Any
OpenTelemetryConfig = Any
Span = Any
class LevoConfig:
"""Configuration for Levo OTLP integration."""
def __init__(
self,
otlp_auth_headers: Optional[str],
protocol: Protocol,
endpoint: str,
):
self.otlp_auth_headers = otlp_auth_headers
self.protocol = protocol
self.endpoint = endpoint
class LevoLogger(OpenTelemetry):
"""Levo Logger that extends OpenTelemetry for OTLP integration."""
@staticmethod
def get_levo_config() -> LevoConfig:
"""
Retrieves the Levo configuration based on environment variables.
Returns:
LevoConfig: Configuration object containing Levo OTLP settings.
Raises:
ValueError: If required environment variables are missing.
"""
# Required environment variables
api_key = os.environ.get("LEVOAI_API_KEY", None)
org_id = os.environ.get("LEVOAI_ORG_ID", None)
workspace_id = os.environ.get("LEVOAI_WORKSPACE_ID", None)
collector_url = os.environ.get("LEVOAI_COLLECTOR_URL", None)
# Validate required env vars
if not api_key:
raise ValueError(
"LEVOAI_API_KEY environment variable is required for Levo integration."
)
if not org_id:
raise ValueError(
"LEVOAI_ORG_ID environment variable is required for Levo integration."
)
if not workspace_id:
raise ValueError(
"LEVOAI_WORKSPACE_ID environment variable is required for Levo integration."
)
if not collector_url:
raise ValueError(
"LEVOAI_COLLECTOR_URL environment variable is required for Levo integration. "
"Please contact Levo support to get your collector URL."
)
# Use collector URL exactly as provided by the user
endpoint = collector_url
protocol: Protocol = "otlp_http"
# Build OTLP headers string
# Format: Authorization=Bearer {api_key},x-levo-organization-id={org_id},x-levo-workspace-id={workspace_id}
headers_parts = [f"Authorization=Bearer {api_key}"]
headers_parts.append(f"x-levo-organization-id={org_id}")
headers_parts.append(f"x-levo-workspace-id={workspace_id}")
otlp_auth_headers = ",".join(headers_parts)
return LevoConfig(
otlp_auth_headers=otlp_auth_headers,
protocol=protocol,
endpoint=endpoint,
)
async def async_health_check(self):
"""
Health check for Levo integration.
Returns:
dict: Health status with status and message/error_message keys.
"""
try:
config = self.get_levo_config()
if not config.otlp_auth_headers:
return {
"status": "unhealthy",
"error_message": "LEVOAI_API_KEY environment variable not set",
}
return {
"status": "healthy",
"message": "Levo credentials are configured properly",
}
except ValueError as e:
return {
"status": "unhealthy",
"error_message": str(e),
}

181
litellm/integrations/opentelemetry.py
View file

@ -48,6 +48,7 @@ else:
LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm")
LITELLM_METER_NAME = os.getenv("LITELLM_METER_NAME", "litellm")
LITELLM_LOGGER_NAME = os.getenv("LITELLM_LOGGER_NAME", "litellm")
LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
# Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
@ -233,14 +234,16 @@ class OpenTelemetry(CustomLogger):
trace.set_tracer_provider(tracer_provider)
else:
# Tracer provider explicitly provided (e.g., for testing)
# Do NOT call set_tracer_provider - the caller is responsible for managing global state
# If they want it to be global, they've already set it before passing it to us
verbose_logger.debug(
"OpenTelemetry: Using provided TracerProvider: %s",
type(tracer_provider).__name__,
)
trace.set_tracer_provider(tracer_provider)
# grab our tracer
self.tracer = trace.get_tracer(LITELLM_TRACER_NAME)
# Grab our tracer from the TracerProvider (not from global context)
# This ensures we use the provided TracerProvider (e.g., for testing)
self.tracer = tracer_provider.get_tracer(LITELLM_TRACER_NAME)
self.span_kind = SpanKind
def _init_metrics(self, meter_provider):
@ -527,6 +530,7 @@ class OpenTelemetry(CustomLogger):
# 3. Guardrail span
self._create_guardrail_span(kwargs=kwargs, context=ctx)
return response
#########################################################
@ -557,9 +561,9 @@ class OpenTelemetry(CustomLogger):
def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]:
"""Extract dynamic headers from kwargs if available."""
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params")
)
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params")
if not standard_callback_dynamic_params:
return None
@ -607,18 +611,35 @@ class OpenTelemetry(CustomLogger):
)
ctx, parent_span = self._get_span_context(kwargs)
if get_secret_bool("USE_OTEL_LITELLM_REQUEST_SPAN"):
primary_span_parent = None
else:
primary_span_parent = parent_span
# 1. Primary span
span = self._start_primary_span(
kwargs, response_obj, start_time, end_time, ctx, primary_span_parent
# Decide whether to create a primary span
# Always create if no parent span exists (backward compatibility)
# OR if USE_OTEL_LITELLM_REQUEST_SPAN is explicitly enabled
should_create_primary_span = parent_span is None or get_secret_bool(
"USE_OTEL_LITELLM_REQUEST_SPAN"
)
# 2. Raw‐request sub-span (if enabled)
self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span)
if should_create_primary_span:
# Create a new litellm_request span
span = self._start_primary_span(
kwargs, response_obj, start_time, end_time, ctx
)
# Raw-request sub-span (if enabled) - child of litellm_request span
self._maybe_log_raw_request(
kwargs, response_obj, start_time, end_time, span
)
else:
# Do not create primary span (keep hierarchy shallow when parent exists)
from opentelemetry.trace import Status, StatusCode
span = None
# Only set attributes if the span is still recording (not closed)
# Note: parent_span is guaranteed to be not None here
parent_span.set_status(Status(StatusCode.OK))
self.set_attributes(parent_span, kwargs, response_obj)
# Raw-request as direct child of parent_span
self._maybe_log_raw_request(
kwargs, response_obj, start_time, end_time, parent_span
)
# 3. Guardrail span
self._create_guardrail_span(kwargs=kwargs, context=ctx)
@ -628,12 +649,18 @@ class OpenTelemetry(CustomLogger):
# 5. Semantic logs.
if self.config.enable_events:
self._emit_semantic_logs(kwargs, response_obj, span)
log_span = span if span is not None else parent_span
if log_span is not None:
self._emit_semantic_logs(kwargs, response_obj, log_span)
# 6. End parent span (only if it wasn't reused as the primary span)
# If parent_span was reused as the primary span, it was already ended in _start_primary_span
if parent_span is not None and parent_span is not span:
parent_span.end(end_time=self._to_ns(datetime.now()))
# 6. Do NOT end parent span - it should be managed by its creator
# External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
# However, proxy-created spans should be closed here
if (
parent_span is not None
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
):
parent_span.end(end_time=self._to_ns(end_time))
def _start_primary_span(
self,
@ -642,16 +669,19 @@ class OpenTelemetry(CustomLogger):
start_time,
end_time,
context,
parent_span: Optional[Span] = None,
):
from opentelemetry.trace import Status, StatusCode
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
span = parent_span or otel_tracer.start_span(
# Always create a new span
# The parent relationship is preserved through the context parameter
span = otel_tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=context,
)
span.set_status(Status(StatusCode.OK))
self.set_attributes(span, kwargs, response_obj)
span.end(end_time=self._to_ns(end_time))
@ -764,10 +794,10 @@ class OpenTelemetry(CustomLogger):
return float(val)
# isinstance(val, str) - parse datetime string (with or without microseconds)
try:
return datetime.strptime(val, '%Y-%m-%d %H:%M:%S.%f').timestamp()
return datetime.strptime(val, "%Y-%m-%d %H:%M:%S.%f").timestamp()
except ValueError:
try:
return datetime.strptime(val, '%Y-%m-%d %H:%M:%S').timestamp()
return datetime.strptime(val, "%Y-%m-%d %H:%M:%S").timestamp()
except ValueError:
return None
@ -775,23 +805,23 @@ class OpenTelemetry(CustomLogger):
"""Record Time to First Token (TTFT) metric for streaming requests."""
optional_params = kwargs.get("optional_params", {})
is_streaming = optional_params.get("stream", False)
if not (self._time_to_first_token_histogram and is_streaming):
return
# Use api_call_start_time for precision (matches Prometheus implementation)
# This excludes LiteLLM overhead and measures pure LLM API latency
api_call_start_time = kwargs.get("api_call_start_time", None)
completion_start_time = kwargs.get("completion_start_time", None)
if api_call_start_time is not None and completion_start_time is not None:
# Convert to timestamps if needed (handles datetime, float, and string)
api_call_start_ts = self._to_timestamp(api_call_start_time)
completion_start_ts = self._to_timestamp(completion_start_time)
if api_call_start_ts is None or completion_start_ts is None:
return # Skip recording if conversion failed
time_to_first_token_seconds = completion_start_ts - api_call_start_ts
self._time_to_first_token_histogram.record(
time_to_first_token_seconds, attributes=common_attrs
@ -806,38 +836,40 @@ class OpenTelemetry(CustomLogger):
common_attrs: dict,
):
"""Record Time Per Output Token (TPOT) metric.
Calculated as: generation_time / completion_tokens
- For streaming: uses end_time - completion_start_time (time to generate all tokens after first)
- For non-streaming: uses end_time - api_call_start_time (total generation time)
"""
if not self._time_per_output_token_histogram:
return
# Get completion tokens from response_obj
completion_tokens = None
if response_obj and (usage := response_obj.get("usage")):
completion_tokens = usage.get("completion_tokens")
if completion_tokens is None or completion_tokens <= 0:
return
# Calculate generation time
completion_start_time = kwargs.get("completion_start_time", None)
api_call_start_time = kwargs.get("api_call_start_time", None)
# Convert end_time to timestamp (handles datetime, float, and string)
end_time_ts = self._to_timestamp(end_time)
if end_time_ts is None:
# Fallback to duration_s if conversion failed
generation_time_seconds = duration_s
if generation_time_seconds > 0:
time_per_output_token_seconds = generation_time_seconds / completion_tokens
time_per_output_token_seconds = (
generation_time_seconds / completion_tokens
)
self._time_per_output_token_histogram.record(
time_per_output_token_seconds, attributes=common_attrs
)
return
if completion_start_time is not None:
# Streaming: use completion_start_time (when first token arrived)
# This measures time to generate all tokens after the first one
@ -858,7 +890,7 @@ class OpenTelemetry(CustomLogger):
else:
# Fallback: use duration_s (already calculated as (end_time - start_time).total_seconds())
generation_time_seconds = duration_s
if generation_time_seconds > 0:
time_per_output_token_seconds = generation_time_seconds / completion_tokens
self._time_per_output_token_histogram.record(
@ -872,37 +904,37 @@ class OpenTelemetry(CustomLogger):
common_attrs: dict,
):
"""Record Total Generation Time (response duration) metric.
Measures pure LLM API generation time: end_time - api_call_start_time
This excludes LiteLLM overhead and measures only the LLM provider's response time.
Works for both streaming and non-streaming requests.
Mirrors Prometheus's litellm_llm_api_latency_metric.
Uses kwargs.get("end_time") with fallback to parameter for consistency with Prometheus.
"""
if not self._response_duration_histogram:
return
api_call_start_time = kwargs.get("api_call_start_time", None)
if api_call_start_time is None:
return
# Use end_time from kwargs if available (matches Prometheus), otherwise use parameter
# For streaming: end_time is when the stream completes (final chunk received)
# For non-streaming: end_time is when the response is received
_end_time = kwargs.get("end_time") or end_time
if _end_time is None:
_end_time = datetime.now()
# Convert to timestamps if needed (handles datetime, float, and string)
api_call_start_ts = self._to_timestamp(api_call_start_time)
end_time_ts = self._to_timestamp(_end_time)
if api_call_start_ts is None or end_time_ts is None:
return # Skip recording if conversion failed
response_duration_seconds = end_time_ts - api_call_start_ts
if response_duration_seconds > 0:
self._response_duration_histogram.record(
response_duration_seconds, attributes=common_attrs
@ -1065,26 +1097,49 @@ class OpenTelemetry(CustomLogger):
)
_parent_context, parent_otel_span = self._get_span_context(kwargs)
# Span 1: Requst sent to litellm SDK
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
span = otel_tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=_parent_context,
# Decide whether to create a primary span
# Always create if no parent span exists (backward compatibility)
# OR if USE_OTEL_LITELLM_REQUEST_SPAN is explicitly enabled
should_create_primary_span = parent_otel_span is None or get_secret_bool(
"USE_OTEL_LITELLM_REQUEST_SPAN"
)
span.set_status(Status(StatusCode.ERROR))
self.set_attributes(span, kwargs, response_obj)
# Record exception information using OTEL standard method
self._record_exception_on_span(span=span, kwargs=kwargs)
if should_create_primary_span:
# Span 1: Request sent to litellm SDK
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
span = otel_tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=_parent_context,
)
span.set_status(Status(StatusCode.ERROR))
self.set_attributes(span, kwargs, response_obj)
span.end(end_time=self._to_ns(end_time))
# Record exception information using OTEL standard method
self._record_exception_on_span(span=span, kwargs=kwargs)
span.end(end_time=self._to_ns(end_time))
else:
# When parent span exists and USE_OTEL_LITELLM_REQUEST_SPAN=false,
# record error on parent span (keeps hierarchy shallow)
# Only set attributes if the span is still recording (not closed)
# Note: parent_otel_span is guaranteed to be not None here
if parent_otel_span.is_recording():
parent_otel_span.set_status(Status(StatusCode.ERROR))
self.set_attributes(parent_otel_span, kwargs, response_obj)
self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
# Create span for guardrail information
self._create_guardrail_span(kwargs=kwargs, context=_parent_context)
if parent_otel_span is not None:
parent_otel_span.end(end_time=self._to_ns(datetime.now()))
# Do NOT end parent span - it should be managed by its creator
# External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
# However, proxy-created spans should be closed here
if (
parent_otel_span is not None
and parent_otel_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
):
parent_otel_span.end(end_time=self._to_ns(end_time))
def _record_exception_on_span(self, span: Span, kwargs: dict):
"""
@ -1263,7 +1318,9 @@ class OpenTelemetry(CustomLogger):
)
return
elif self.callback_name == "weave_otel":
from litellm.integrations.weave.weave_otel import set_weave_otel_attributes
from litellm.integrations.weave.weave_otel import (
set_weave_otel_attributes,
)
set_weave_otel_attributes(span, kwargs, response_obj)
return
@ -1994,9 +2051,9 @@ class OpenTelemetry(CustomLogger):
"""
Create a span for the received proxy server request.
"""
return self.tracer.start_span(
name="Received Proxy Server Request",
name=LITELLM_PROXY_REQUEST_SPAN_NAME,
start_time=self._to_ns(start_time),
context=self.get_traceparent_from_header(headers=headers),
kind=self.span_kind.SERVER,

4
litellm/integrations/prometheus.py
View file

@ -214,7 +214,7 @@ class PrometheusLogger(CustomLogger):
# Remaining Rate Limit for model
self.litellm_remaining_requests_metric = self._gauge_factory(
"litellm_remaining_requests",
"litellm_remaining_requests_metric",
"LLM Deployment Analytics - remaining requests for model, returned from LLM API Provider",
labelnames=self.get_labels_for_metric(
"litellm_remaining_requests_metric"
@ -222,7 +222,7 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_remaining_tokens_metric = self._gauge_factory(
"litellm_remaining_tokens",
"litellm_remaining_tokens_metric",
"remaining tokens for model, returned from LLM API Provider",
labelnames=self.get_labels_for_metric(
"litellm_remaining_tokens_metric"

72
litellm/litellm_core_utils/core_helpers.py
View file

@ -38,18 +38,18 @@ def safe_divide_seconds(
def safe_divide(
numerator: Union[int, float],
denominator: Union[int, float],
default: Union[int, float] = 0
numerator: Union[int, float],
denominator: Union[int, float],
default: Union[int, float] = 0,
) -> Union[int, float]:
"""
Safely divide two numbers, returning a default value if denominator is zero.
Args:
numerator: The number to divide
denominator: The number to divide by
default: Value to return if denominator is zero (defaults to 0)
Returns:
The result of numerator/denominator, or default if denominator is zero
"""
@ -153,7 +153,8 @@ def get_metadata_variable_name_from_kwargs(
- LiteLLM is now moving to using `litellm_metadata` for our metadata
"""
return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
def get_litellm_metadata_from_kwargs(kwargs: dict):
"""
Helper to get litellm metadata from all litellm request kwargs
@ -176,6 +177,25 @@ def get_litellm_metadata_from_kwargs(kwargs: dict):
return {}
def reconstruct_model_name(
model_name: str,
custom_llm_provider: Optional[str],
metadata: dict,
) -> str:
"""Reconstruct full model name with provider prefix for logging."""
# Check if deployment model name from router metadata is available (has original prefix)
deployment_model_name = metadata.get("deployment")
if deployment_model_name and "/" in deployment_model_name:
# Use the deployment model name which preserves the original provider prefix
return deployment_model_name
elif custom_llm_provider and model_name and "/" not in model_name:
# Only add prefix for Bedrock (not for direct Anthropic API)
# This ensures Bedrock models get the prefix while direct Anthropic models don't
if custom_llm_provider == "bedrock":
return f"{custom_llm_provider}/{model_name}"
return model_name
# Helper functions used for OTEL logging
def _get_parent_otel_span_from_kwargs(
kwargs: Optional[dict] = None,
@ -246,8 +266,8 @@ def safe_deep_copy(data):
Safe Deep Copy
The LiteLLM request may contain objects that cannot be pickled/deep-copied
(e.g., tracing spans, locks, clients).
(e.g., tracing spans, locks, clients).
This helper deep-copies each top-level key independently; on failure keeps
original ref
"""
@ -306,23 +326,23 @@ def safe_deep_copy(data):
def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
"""
Recursively filter out Exception objects and callable objects from dicts/lists.
This is a defensive utility to prevent deepcopy failures when exception objects
are accidentally stored in parameter dictionaries (e.g., optional_params).
Also filters callable objects (functions) to prevent JSON serialization errors.
Exceptions and callables should not be stored in params - this function removes them.
Args:
data: The data structure to filter (dict, list, or any other type)
max_depth: Maximum recursion depth to prevent infinite loops
Returns:
Filtered data structure with Exception and callable objects removed, or None if the
entire input was an Exception or callable
"""
if max_depth <= 0:
return data
# Skip exception objects
if isinstance(data, Exception):
return None
@ -333,7 +353,7 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
obj_type_name = type(data).__name__
if obj_type_name in ["Logging", "LiteLLMLoggingObj"]:
return None
if isinstance(data, dict):
result: dict[str, Any] = {}
for k, v in data.items():
@ -352,7 +372,9 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
result_list: list[Any] = []
for item in data:
# Skip exception and callable items
if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)):
if isinstance(item, Exception) or (
callable(item) and not isinstance(item, type)
):
continue
try:
filtered = filter_exceptions_from_params(item, max_depth - 1)
@ -366,37 +388,35 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
return data
def filter_internal_params(data: dict, additional_internal_params: Optional[set] = None) -> dict:
def filter_internal_params(
data: dict, additional_internal_params: Optional[set] = None
) -> dict:
"""
Filter out LiteLLM internal parameters that shouldn't be sent to provider APIs.
This removes internal/MCP-related parameters that are used by LiteLLM internally
but should not be included in API requests to providers.
Args:
data: Dictionary of parameters to filter
additional_internal_params: Optional set of additional internal parameter names to filter
Returns:
Filtered dictionary with internal parameters removed
"""
if not isinstance(data, dict):
return data
# Known internal parameters that should never be sent to provider APIs
internal_params = {
"skip_mcp_handler",
"mcp_handler_context",
"_skip_mcp_handler",
}
# Add any additional internal params if provided
if additional_internal_params:
internal_params.update(additional_internal_params)
# Filter out internal parameters
return {
k: v
for k, v in data.items()
if k not in internal_params
}
return {k: v for k, v in data.items() if k not in internal_params}

1
litellm/litellm_core_utils/custom_logger_registry.py
View file

@ -76,6 +76,7 @@ class CustomLoggerRegistry:
"arize_phoenix": OpenTelemetry,
"langtrace": OpenTelemetry,
"weave_otel": OpenTelemetry,
"levo": OpenTelemetry,
"mlflow": MlflowLogger,
"langfuse": LangfusePromptManagement,
"otel": OpenTelemetry,

11
litellm/litellm_core_utils/dot_notation_indexing.py
View file

@ -9,6 +9,7 @@ Custom implementation with zero external dependencies.
Supported syntax:
- "field" - top-level field
- "parent.child" - nested field
- "parent\\.with\\.dots.child" - keys containing dots (escape with backslash)
- "array[*]" - all array elements (wildcard)
- "array[0]" - specific array element (index)
- "array[*].field" - field in all array elements
@ -47,6 +48,9 @@ def get_nested_value(
'value'
>>> get_nested_value(data, "a.b.d", "default")
'default'
>>> data = {"kubernetes.io": {"namespace": "default"}}
>>> get_nested_value(data, "kubernetes\\.io.namespace")
'default'
"""
if not key_path:
return default
@ -58,8 +62,11 @@ def get_nested_value(
else key_path
)
# Split the key path into parts
parts = key_path.split(".")
# Split the key path into parts, respecting escaped dots (\.)
# Use a temporary placeholder, split on unescaped dots, then restore
placeholder = "\x00"
parts = key_path.replace("\\.", placeholder).split(".")
parts = [p.replace(placeholder, ".") for p in parts]
# Traverse through the dictionary
current: Any = data

4
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -229,10 +229,10 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "https://api.ai21.com/studio/v1":
custom_llm_provider = "ai21_chat"
dynamic_api_key = get_secret_str("AI21_API_KEY")
elif endpoint == "https://codestral.mistral.ai/v1":
elif endpoint == "codestral.mistral.ai/v1/chat/completions":
custom_llm_provider = "codestral"
dynamic_api_key = get_secret_str("CODESTRAL_API_KEY")
elif endpoint == "https://codestral.mistral.ai/v1":
elif endpoint == "codestral.mistral.ai/v1/fim/completions":
custom_llm_provider = "text-completion-codestral"
dynamic_api_key = get_secret_str("CODESTRAL_API_KEY")
elif endpoint == "app.empower.dev/api/v1":

376
litellm/litellm_core_utils/litellm_logging.py
View file

@ -59,6 +59,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.core_helpers import reconstruct_model_name
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
@ -332,9 +333,9 @@ class Logging(LiteLLMLoggingBaseClass):
self.litellm_trace_id: str = litellm_trace_id or str(uuid.uuid4())
self.function_id = function_id
self.streaming_chunks: List[Any] = [] # for generating complete stream response
self.sync_streaming_chunks: List[Any] = (
[]
) # for generating complete stream response
self.sync_streaming_chunks: List[
Any
] = [] # for generating complete stream response
self.log_raw_request_response = log_raw_request_response
# Initialize dynamic callbacks
@ -719,9 +720,9 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_spec=prompt_spec,
dynamic_callback_params=dynamic_callback_params,
):
self.model_call_details["prompt_integration"] = (
logger.__class__.__name__
)
self.model_call_details[
"prompt_integration"
] = logger.__class__.__name__
return logger
except Exception:
# If check fails, continue to next logger
@ -789,9 +790,9 @@ class Logging(LiteLLMLoggingBaseClass):
if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
non_default_params
):
self.model_call_details["prompt_integration"] = (
anthropic_cache_control_logger.__class__.__name__
)
self.model_call_details[
"prompt_integration"
] = anthropic_cache_control_logger.__class__.__name__
return anthropic_cache_control_logger
#########################################################
@ -803,9 +804,9 @@ class Logging(LiteLLMLoggingBaseClass):
internal_usage_cache=None,
llm_router=None,
)
self.model_call_details["prompt_integration"] = (
vector_store_custom_logger.__class__.__name__
)
self.model_call_details[
"prompt_integration"
] = vector_store_custom_logger.__class__.__name__
# Add to global callbacks so post-call hooks are invoked
if (
vector_store_custom_logger
@ -865,9 +866,9 @@ class Logging(LiteLLMLoggingBaseClass):
model
): # if model name was changes pre-call, overwrite the initial model call name with the new one
self.model_call_details["model"] = model
self.model_call_details["litellm_params"]["api_base"] = (
self._get_masked_api_base(additional_args.get("api_base", ""))
)
self.model_call_details["litellm_params"][
"api_base"
] = self._get_masked_api_base(additional_args.get("api_base", ""))
def pre_call(self, input, api_key, model=None, additional_args={}): # noqa: PLR0915
# Log the exact input to the LLM API
@ -896,10 +897,10 @@ class Logging(LiteLLMLoggingBaseClass):
try:
# [Non-blocking Extra Debug Information in metadata]
if turn_off_message_logging is True:
_metadata["raw_request"] = (
"redacted by litellm. \
_metadata[
"raw_request"
] = "redacted by litellm. \
'litellm.turn_off_message_logging=True'"
)
else:
curl_command = self._get_request_curl_command(
api_base=additional_args.get("api_base", ""),
@ -910,34 +911,34 @@ class Logging(LiteLLMLoggingBaseClass):
_metadata["raw_request"] = str(curl_command)
# split up, so it's easier to parse in the UI
self.model_call_details["raw_request_typed_dict"] = (
RawRequestTypedDict(
raw_request_api_base=str(
additional_args.get("api_base") or ""
),
raw_request_body=self._get_raw_request_body(
additional_args.get("complete_input_dict", {})
),
# NOTE: setting ignore_sensitive_headers to True will cause
# the Authorization header to be leaked when calls to the health
# endpoint are made and fail.
raw_request_headers=self._get_masked_headers(
additional_args.get("headers", {}) or {},
),
error=None,
)
self.model_call_details[
"raw_request_typed_dict"
] = RawRequestTypedDict(
raw_request_api_base=str(
additional_args.get("api_base") or ""
),
raw_request_body=self._get_raw_request_body(
additional_args.get("complete_input_dict", {})
),
# NOTE: setting ignore_sensitive_headers to True will cause
# the Authorization header to be leaked when calls to the health
# endpoint are made and fail.
raw_request_headers=self._get_masked_headers(
additional_args.get("headers", {}) or {},
),
error=None,
)
except Exception as e:
self.model_call_details["raw_request_typed_dict"] = (
RawRequestTypedDict(
error=str(e),
)
self.model_call_details[
"raw_request_typed_dict"
] = RawRequestTypedDict(
error=str(e),
)
_metadata["raw_request"] = (
"Unable to Log \
_metadata[
"raw_request"
] = "Unable to Log \
raw request: {}".format(
str(e)
)
str(e)
)
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
@ -1238,13 +1239,13 @@ class Logging(LiteLLMLoggingBaseClass):
for callback in callbacks:
try:
if isinstance(callback, CustomLogger):
response: Optional[MCPPostCallResponseObject] = (
await callback.async_post_mcp_tool_call_hook(
kwargs=kwargs,
response_obj=post_mcp_tool_call_response_obj,
start_time=start_time,
end_time=end_time,
)
response: Optional[
MCPPostCallResponseObject
] = await callback.async_post_mcp_tool_call_hook(
kwargs=kwargs,
response_obj=post_mcp_tool_call_response_obj,
start_time=start_time,
end_time=end_time,
)
######################################################################
# if any of the callbacks modify the response, use the modified response
@ -1423,9 +1424,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
f"response_cost_failure_debug_information: {debug_info}"
)
self.model_call_details["response_cost_failure_debug_information"] = (
debug_info
)
self.model_call_details[
"response_cost_failure_debug_information"
] = debug_info
return None
try:
@ -1451,9 +1452,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
f"response_cost_failure_debug_information: {debug_info}"
)
self.model_call_details["response_cost_failure_debug_information"] = (
debug_info
)
self.model_call_details[
"response_cost_failure_debug_information"
] = debug_info
return None
@ -1603,16 +1604,16 @@ class Logging(LiteLLMLoggingBaseClass):
result=logging_result
)
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
def _transform_usage_objects(self, result):
@ -1667,9 +1668,9 @@ class Logging(LiteLLMLoggingBaseClass):
end_time = datetime.datetime.now()
if self.completion_start_time is None:
self.completion_start_time = end_time
self.model_call_details["completion_start_time"] = (
self.completion_start_time
)
self.model_call_details[
"completion_start_time"
] = self.completion_start_time
self.model_call_details["log_event_type"] = "successful_api_call"
self.model_call_details["end_time"] = end_time
@ -1706,21 +1707,21 @@ class Logging(LiteLLMLoggingBaseClass):
end_time=end_time,
)
elif isinstance(result, dict) or isinstance(result, list):
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
elif standard_logging_object is not None:
self.model_call_details["standard_logging_object"] = (
standard_logging_object
)
self.model_call_details[
"standard_logging_object"
] = standard_logging_object
else:
self.model_call_details["response_cost"] = None
@ -1870,23 +1871,23 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
"Logging Details LiteLLM-Success Call streaming complete"
)
self.model_call_details["complete_streaming_response"] = (
complete_streaming_response
)
self.model_call_details["response_cost"] = (
self._response_cost_calculator(result=complete_streaming_response)
)
self.model_call_details[
"complete_streaming_response"
] = complete_streaming_response
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(result=complete_streaming_response)
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
callbacks = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_success_callbacks,
@ -2214,10 +2215,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
else:
if self.stream and complete_streaming_response:
self.model_call_details["complete_response"] = (
self.model_call_details.get(
"complete_streaming_response", {}
)
self.model_call_details[
"complete_response"
] = self.model_call_details.get(
"complete_streaming_response", {}
)
result = self.model_call_details["complete_response"]
openMeterLogger.log_success_event(
@ -2256,10 +2257,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
else:
if self.stream and complete_streaming_response:
self.model_call_details["complete_response"] = (
self.model_call_details.get(
"complete_streaming_response", {}
)
self.model_call_details[
"complete_response"
] = self.model_call_details.get(
"complete_streaming_response", {}
)
result = self.model_call_details["complete_response"]
@ -2402,9 +2403,9 @@ class Logging(LiteLLMLoggingBaseClass):
if complete_streaming_response is not None:
print_verbose("Async success callbacks: Got a complete streaming response")
self.model_call_details["async_complete_streaming_response"] = (
complete_streaming_response
)
self.model_call_details[
"async_complete_streaming_response"
] = complete_streaming_response
try:
if self.model_call_details.get("cache_hit", False) is True:
@ -2415,10 +2416,10 @@ class Logging(LiteLLMLoggingBaseClass):
model_call_details=self.model_call_details
)
# base_model defaults to None if not set on model_info
self.model_call_details["response_cost"] = (
self._response_cost_calculator(
result=complete_streaming_response
)
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(
result=complete_streaming_response
)
verbose_logger.debug(
@ -2431,16 +2432,16 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = None
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
callbacks = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_async_success_callbacks,
@ -2676,18 +2677,18 @@ class Logging(LiteLLMLoggingBaseClass):
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj={},
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="failure",
error_str=str(exception),
original_exception=exception,
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj={},
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="failure",
error_str=str(exception),
original_exception=exception,
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
return start_time, end_time
@ -3301,7 +3302,9 @@ class Logging(LiteLLMLoggingBaseClass):
# Deep copy result and add usage
result_copy = result.model_copy(deep=True)
result_copy.usage = usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)
result_copy.usage = (
usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)
)
return result_copy
@ -3629,9 +3632,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
endpoint=arize_config.endpoint,
)
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
for callback in _in_memory_loggers:
if (
isinstance(callback, ArizeLogger)
@ -3642,7 +3645,6 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
_in_memory_loggers.append(_arize_otel_logger)
return _arize_otel_logger # type: ignore
elif logging_integration == "arize_phoenix":
from litellm.integrations.opentelemetry import (
OpenTelemetry,
OpenTelemetryConfig,
@ -3658,13 +3660,13 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
# Add openinference.project.name attribute
if existing_attrs:
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
)
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
else:
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"openinference.project.name={arize_phoenix_config.project_name}"
)
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"openinference.project.name={arize_phoenix_config.project_name}"
# Set Phoenix project name from environment variable
phoenix_project_name = os.environ.get("PHOENIX_PROJECT_NAME", None)
@ -3672,19 +3674,19 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
# Add openinference.project.name attribute
if existing_attrs:
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"{existing_attrs},openinference.project.name={phoenix_project_name}"
)
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"{existing_attrs},openinference.project.name={phoenix_project_name}"
else:
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"openinference.project.name={phoenix_project_name}"
)
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"openinference.project.name={phoenix_project_name}"
# auth can be disabled on local deployments of arize phoenix
if arize_phoenix_config.otlp_auth_headers is not None:
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
arize_phoenix_config.otlp_auth_headers
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = arize_phoenix_config.otlp_auth_headers
for callback in _in_memory_loggers:
if (
@ -3697,6 +3699,31 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
)
_in_memory_loggers.append(_arize_phoenix_otel_logger)
return _arize_phoenix_otel_logger # type: ignore
elif logging_integration == "levo":
from litellm.integrations.levo.levo import LevoLogger
from litellm.integrations.opentelemetry import (
OpenTelemetry,
OpenTelemetryConfig,
)
levo_config = LevoLogger.get_levo_config()
otel_config = OpenTelemetryConfig(
exporter=levo_config.protocol,
endpoint=levo_config.endpoint,
headers=levo_config.otlp_auth_headers,
)
# Check if LevoLogger instance already exists
for callback in _in_memory_loggers:
if (
isinstance(callback, LevoLogger)
and callback.callback_name == "levo"
):
return callback # type: ignore
_levo_otel_logger = LevoLogger(config=otel_config, callback_name="levo")
_in_memory_loggers.append(_levo_otel_logger)
return _levo_otel_logger # type: ignore
elif logging_integration == "otel":
from litellm.integrations.opentelemetry import OpenTelemetry
@ -3816,9 +3843,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
exporter="otlp_http",
endpoint="https://langtrace.ai/api/trace",
)
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
f"api_key={os.getenv('LANGTRACE_API_KEY')}"
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
for callback in _in_memory_loggers:
if (
isinstance(callback, OpenTelemetry)
@ -4589,10 +4616,10 @@ class StandardLoggingPayloadSetup:
for key in StandardLoggingHiddenParams.__annotations__.keys():
if key in hidden_params:
if key == "additional_headers":
clean_hidden_params["additional_headers"] = (
StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
)
clean_hidden_params[
"additional_headers"
] = StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
)
else:
clean_hidden_params[key] = hidden_params[key] # type: ignore
@ -4898,25 +4925,6 @@ def _extract_response_obj_and_hidden_params(
return response_obj, hidden_params
def _reconstruct_model_name(
model_name: str,
custom_llm_provider: Optional[str],
metadata: dict,
) -> str:
"""Reconstruct full model name with provider prefix for logging."""
# Check if deployment model name from router metadata is available (has original prefix)
deployment_model_name = metadata.get("deployment")
if deployment_model_name and "/" in deployment_model_name:
# Use the deployment model name which preserves the original provider prefix
return deployment_model_name
elif custom_llm_provider and model_name and "/" not in model_name:
# Only add prefix for Bedrock (not for direct Anthropic API)
# This ensures Bedrock models get the prefix while direct Anthropic models don't
if custom_llm_provider == "bedrock":
return f"{custom_llm_provider}/{model_name}"
return model_name
def get_standard_logging_object_payload(
kwargs: Optional[dict],
init_response_obj: Union[Any, BaseModel, dict],
@ -5049,7 +5057,7 @@ def get_standard_logging_object_payload(
# This ensures Bedrock models like "us.anthropic.claude-3-5-sonnet-20240620-v1:0"
# are logged as "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
custom_llm_provider = cast(Optional[str], kwargs.get("custom_llm_provider"))
model_name = _reconstruct_model_name(
model_name = reconstruct_model_name(
kwargs.get("model", "") or "", custom_llm_provider, metadata
)
@ -5205,9 +5213,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
):
for k, v in metadata["user_api_key_metadata"].items():
if k == "logging": # prevent logging user logging keys
cleaned_user_api_key_metadata[k] = (
"scrubbed_by_litellm_for_sensitive_keys"
)
cleaned_user_api_key_metadata[
k
] = "scrubbed_by_litellm_for_sensitive_keys"
else:
cleaned_user_api_key_metadata[k] = v

85
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -161,6 +161,15 @@ def _get_token_base_cost(
prompt_base_cost = cast(float, _get_cost_per_unit(model_info, input_cost_key))
completion_base_cost = cast(float, _get_cost_per_unit(model_info, output_cost_key))
# For image generation models that don't have output_cost_per_token,
# use output_cost_per_image_token as the base cost (all output tokens are image tokens)
if completion_base_cost == 0.0 or completion_base_cost is None:
output_image_cost = _get_cost_per_unit(
model_info, "output_cost_per_image_token", None
)
if output_image_cost is not None:
completion_base_cost = cast(float, output_image_cost)
cache_creation_cost = cast(
float, _get_cost_per_unit(model_info, cache_creation_cost_key)
)
@ -342,6 +351,7 @@ class PromptTokensDetailsResult(TypedDict):
cache_creation_token_details: Optional[CacheCreationTokenDetails]
text_tokens: int
audio_tokens: int
image_tokens: int
character_count: int
image_count: int
video_length_seconds: int
@ -374,6 +384,10 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
or 0
)
image_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "image_tokens", 0))
or 0
)
character_count = (
cast(
Optional[int],
@ -398,6 +412,7 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,
audio_tokens=audio_tokens,
image_tokens=image_tokens,
character_count=character_count,
image_count=image_count,
video_length_seconds=video_length_seconds,
@ -470,6 +485,11 @@ def _calculate_input_cost(
model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"]
)
### IMAGE TOKEN COST (for gpt-image-1 and similar models)
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image_token", prompt_tokens_details["image_tokens"]
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
@ -533,6 +553,7 @@ def generic_cost_per_token(
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
audio_tokens=0,
image_tokens=0,
character_count=0,
image_count=0,
video_length_seconds=0,
@ -583,12 +604,22 @@ def generic_cost_per_token(
reasoning_tokens = completion_tokens_details["reasoning_tokens"]
image_tokens = completion_tokens_details["image_tokens"]
# Only assume all tokens are text if there's NO breakdown at all
# If image_tokens, audio_tokens, or reasoning_tokens exist, respect text_tokens=0
# Handle text_tokens calculation:
# 1. If text_tokens is explicitly provided and > 0, use it
# 2. If there's a breakdown (reasoning/audio/image tokens), calculate text_tokens as the remainder
# 3. If no breakdown at all, assume all completion_tokens are text_tokens
has_token_breakdown = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0
if text_tokens == 0 and not has_token_breakdown:
text_tokens = usage.completion_tokens
is_text_tokens_total = True
if text_tokens == 0:
if has_token_breakdown:
# Calculate text tokens as remainder when we have a breakdown
# This handles cases like OpenAI's reasoning models where text_tokens isn't provided
text_tokens = max(
0, usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens
)
else:
# No breakdown at all, all tokens are text tokens
text_tokens = usage.completion_tokens
is_text_tokens_total = True
## TEXT COST
completion_cost = float(text_tokens) * completion_base_cost
@ -782,6 +813,50 @@ class CostCalculatorUtils:
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.OPENAI.value:
# Check if this is a gpt-image model (token-based pricing)
model_lower = model.lower()
if "gpt-image-1" in model_lower:
from litellm.llms.openai.image_generation.cost_calculator import (
cost_calculator as openai_gpt_image_cost_calculator,
)
return openai_gpt_image_cost_calculator(
model=model,
image_response=completion_response,
custom_llm_provider=custom_llm_provider,
)
# Fall through to default for DALL-E models
return default_image_cost_calculator(
model=model,
quality=quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
optional_params=optional_params,
)
elif custom_llm_provider == litellm.LlmProviders.AZURE.value:
# Check if this is a gpt-image model (token-based pricing)
model_lower = model.lower()
if "gpt-image-1" in model_lower:
from litellm.llms.openai.image_generation.cost_calculator import (
cost_calculator as openai_gpt_image_cost_calculator,
)
return openai_gpt_image_cost_calculator(
model=model,
image_response=completion_response,
custom_llm_provider=custom_llm_provider,
)
# Fall through to default for DALL-E models
return default_image_cost_calculator(
model=model,
quality=quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
optional_params=optional_params,
)
else:
return default_image_cost_calculator(
model=model,

46
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -445,25 +445,43 @@ def convert_to_model_response_object( # noqa: PLR0915
hidden_params["additional_headers"] = additional_headers
### CHECK IF ERROR IN RESPONSE ### - openrouter returns these in the dictionary
# Some OpenAI-compatible providers (e.g., Apertis) return empty error objects
# even on success. Only raise if the error contains meaningful data.
if (
response_object is not None
and "error" in response_object
and response_object["error"] is not None
):
error_args = {"status_code": 422, "message": "Error in response object"}
if isinstance(response_object["error"], dict):
if "code" in response_object["error"]:
error_args["status_code"] = response_object["error"]["code"]
if "message" in response_object["error"]:
if isinstance(response_object["error"]["message"], dict):
message_str = json.dumps(response_object["error"]["message"])
else:
message_str = str(response_object["error"]["message"])
error_args["message"] = message_str
raised_exception = Exception()
setattr(raised_exception, "status_code", error_args["status_code"])
setattr(raised_exception, "message", error_args["message"])
raise raised_exception
error_obj = response_object["error"]
has_meaningful_error = False
if isinstance(error_obj, dict):
# Check if error dict has non-empty message or non-null code
error_message = error_obj.get("message", "")
error_code = error_obj.get("code")
has_meaningful_error = bool(error_message) or error_code is not None
elif isinstance(error_obj, str):
# String error is meaningful if non-empty
has_meaningful_error = bool(error_obj)
else:
# Any other truthy value is considered meaningful
has_meaningful_error = True
if has_meaningful_error:
error_args = {"status_code": 422, "message": "Error in response object"}
if isinstance(error_obj, dict):
if "code" in error_obj:
error_args["status_code"] = error_obj["code"]
if "message" in error_obj:
if isinstance(error_obj["message"], dict):
message_str = json.dumps(error_obj["message"])
else:
message_str = str(error_obj["message"])
error_args["message"] = message_str
raised_exception = Exception()
setattr(raised_exception, "status_code", error_args["status_code"])
setattr(raised_exception, "message", error_args["message"])
raise raised_exception
try:
if response_type == "completion" and (

3
litellm/litellm_core_utils/logging_callback_manager.py
View file

@ -166,6 +166,7 @@ class LoggingCallbackManager:
endpoint = callback_config.get("endpoint")
headers = callback_config.get("headers")
event_types = callback_config.get("event_types")
log_format = callback_config.get("log_format")
if endpoint is None or headers is None:
verbose_logger.warning(
@ -180,6 +181,7 @@ class LoggingCallbackManager:
and cached_logger.endpoint == endpoint
and cached_logger.headers == headers
and cached_logger.event_types == event_types
and cached_logger.log_format == log_format
):
return cached_logger
@ -187,6 +189,7 @@ class LoggingCallbackManager:
endpoint=endpoint,
headers=headers,
event_types=event_types,
log_format=log_format,
)
_generic_api_logger_cache[callback] = new_logger
return new_logger

96
litellm/litellm_core_utils/logging_worker.py
View file

@ -51,6 +51,7 @@ class LoggingWorker:
self._worker_task: Optional[asyncio.Task] = None
self._running_tasks: set[asyncio.Task] = set()
self._sem: Optional[asyncio.Semaphore] = None
self._bound_loop: Optional[asyncio.AbstractEventLoop] = None
self._last_aggressive_clear_time: float = 0.0
self._aggressive_clear_in_progress: bool = False
@ -58,9 +59,27 @@ class LoggingWorker:
atexit.register(self._flush_on_exit)
def _ensure_queue(self) -> None:
"""Initialize the queue if it doesn't exist."""
"""Initialize the queue if it doesn't exist or if event loop has changed."""
try:
current_loop = asyncio.get_running_loop()
except RuntimeError:
# No running loop, can't initialize
return
# Check if we need to reinitialize due to event loop change
if self._queue is not None and self._bound_loop is not current_loop:
verbose_logger.debug(
"LoggingWorker: Event loop changed, reinitializing queue and worker"
)
# Clear old state - these are bound to the old loop
self._queue = None
self._sem = None
self._worker_task = None
self._running_tasks.clear()
if self._queue is None:
self._queue = asyncio.Queue(maxsize=self.max_queue_size)
self._bound_loop = current_loop
def start(self) -> None:
"""Start the logging worker. Idempotent - safe to call multiple times."""
@ -126,7 +145,7 @@ class LoggingWorker:
# Capture the current context when enqueueing
task = LoggingTask(coroutine=coroutine, context=contextvars.copy_context())
try:
self._queue.put_nowait(task)
except asyncio.QueueFull:
@ -141,15 +160,15 @@ class LoggingWorker:
"""
if self._aggressive_clear_in_progress:
return False
try:
loop = asyncio.get_running_loop()
current_time = loop.time()
time_since_last_clear = current_time - self._last_aggressive_clear_time
if time_since_last_clear < LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS:
return False
return True
except RuntimeError:
# No event loop running, drop the task
@ -158,8 +177,8 @@ class LoggingWorker:
def _mark_aggressive_clear_started(self) -> None:
"""
Mark that an aggressive clear operation has started.
Note: This should only be called after _should_start_aggressive_clear()
Note: This should only be called after _should_start_aggressive_clear()
returns True, which guarantees an event loop exists.
"""
loop = asyncio.get_running_loop()
@ -171,7 +190,7 @@ class LoggingWorker:
Handle queue full condition by either starting an aggressive clear
or scheduling a delayed retry.
"""
if self._should_start_aggressive_clear():
self._mark_aggressive_clear_started()
# Schedule clearing as async task so enqueue returns immediately (non-blocking)
@ -191,7 +210,8 @@ class LoggingWorker:
time_since_last_clear = current_time - self._last_aggressive_clear_time
remaining_cooldown = max(
0.0,
LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS - time_since_last_clear
LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS
- time_since_last_clear,
)
# Add a small buffer (10% of cooldown or 50ms, whichever is larger) to ensure
# cooldown has expired and aggressive clear has completed
@ -212,7 +232,7 @@ class LoggingWorker:
# Check that we have a running event loop (will raise RuntimeError if not)
asyncio.get_running_loop()
delay = self._calculate_retry_delay()
# Schedule the retry as a background task
asyncio.create_task(self._retry_enqueue_task(task, delay))
except RuntimeError:
@ -225,11 +245,11 @@ class LoggingWorker:
This is called as a background task from _schedule_delayed_enqueue_retry.
"""
await asyncio.sleep(delay)
# Try to enqueue the task directly, preserving its original context
if self._queue is None:
return
try:
self._queue.put_nowait(task)
except asyncio.QueueFull:
@ -243,15 +263,17 @@ class LoggingWorker:
"""
if self._queue is None:
return []
# Calculate items based on percentage of queue size
items_to_extract = (self.max_queue_size * LOGGING_WORKER_CLEAR_PERCENTAGE) // 100
items_to_extract = (
self.max_queue_size * LOGGING_WORKER_CLEAR_PERCENTAGE
) // 100
# Use actual queue size to avoid unnecessary iterations
actual_size = self._queue.qsize()
if actual_size == 0:
return []
items_to_extract = min(items_to_extract, actual_size)
# Extract tasks from queue (using list comprehension would require wrapping in try/except)
extracted_tasks = []
for _ in range(items_to_extract):
@ -259,10 +281,12 @@ class LoggingWorker:
extracted_tasks.append(self._queue.get_nowait())
except asyncio.QueueEmpty:
break
return extracted_tasks
async def _aggressively_clear_queue_async(self, new_task: Optional[LoggingTask] = None) -> None:
async def _aggressively_clear_queue_async(
self, new_task: Optional[LoggingTask] = None
) -> None:
"""
Aggressively clear the queue by extracting and processing items.
This is called when the queue is full to prevent dropping logs.
@ -271,18 +295,20 @@ class LoggingWorker:
try:
if self._queue is None:
return
extracted_tasks = self._extract_tasks_from_queue()
# Add new task to extracted tasks to process directly
if new_task is not None:
extracted_tasks.append(new_task)
# Process extracted tasks directly
if extracted_tasks:
await self._process_extracted_tasks(extracted_tasks)
except Exception as e:
verbose_logger.exception(f"LoggingWorker error during aggressive clear: {e}")
verbose_logger.exception(
f"LoggingWorker error during aggressive clear: {e}"
)
finally:
# Always reset the flag even if an error occurs
self._aggressive_clear_in_progress = False
@ -291,7 +317,7 @@ class LoggingWorker:
"""Process a single task and mark it done."""
if self._queue is None:
return
try:
await asyncio.wait_for(
task["context"].run(asyncio.create_task, task["coroutine"]),
@ -310,7 +336,7 @@ class LoggingWorker:
"""
if not tasks or self._queue is None:
return
# Process all tasks concurrently for maximum speed
await asyncio.gather(*[self._process_single_task(task) for task in tasks])
@ -361,10 +387,7 @@ class LoggingWorker:
for _ in range(MAX_ITERATIONS_TO_CLEAR_QUEUE):
# Check if we've exceeded the maximum time
if (
asyncio.get_event_loop().time() - start_time
>= MAX_TIME_TO_CLEAR_QUEUE
):
if asyncio.get_event_loop().time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE:
verbose_logger.warning(
f"clear_queue exceeded max_time of {MAX_TIME_TO_CLEAR_QUEUE}s, stopping early"
)
@ -381,6 +404,9 @@ class LoggingWorker:
except Exception:
# Suppress errors during cleanup
pass
finally:
# Clear reference to prevent memory leaks
task = None
self._queue.task_done() # If you're using join() elsewhere
except asyncio.QueueEmpty:
break
@ -410,7 +436,7 @@ class LoggingWorker:
This ensures callbacks queued by async completions are processed
even when the script exits before the worker loop can handle them.
Note: All logging in this method is wrapped to handle cases where
logging handlers are closed during shutdown.
"""
@ -423,7 +449,9 @@ class LoggingWorker:
return
queue_size = self._queue.qsize()
self._safe_log("info", f"[LoggingWorker] atexit: Flushing {queue_size} remaining events...")
self._safe_log(
"info", f"[LoggingWorker] atexit: Flushing {queue_size} remaining events..."
)
# Create a new event loop since the original is closed
loop = asyncio.new_event_loop()
@ -438,7 +466,7 @@ class LoggingWorker:
if loop.time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE:
self._safe_log(
"warning",
f"[LoggingWorker] atexit: Reached time limit ({MAX_TIME_TO_CLEAR_QUEUE}s), stopping flush"
f"[LoggingWorker] atexit: Reached time limit ({MAX_TIME_TO_CLEAR_QUEUE}s), stopping flush",
)
break
@ -456,8 +484,14 @@ class LoggingWorker:
except Exception:
# Silent failure to not break user's program
pass
finally:
# Clear reference to prevent memory leaks
task = None
self._safe_log("info", f"[LoggingWorker] atexit: Successfully flushed {processed} events!")
self._safe_log(
"info",
f"[LoggingWorker] atexit: Successfully flushed {processed} events!",
)
finally:
loop.close()

32
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -1087,9 +1087,35 @@ def _parse_content_for_reasoning(
return None, message_text
def _extract_base64_data(image_url: str) -> str:
"""
Extract pure base64 data from an image URL.
If the URL is a data URL (e.g., "data:image/png;base64,iVBOR..."),
extract and return only the base64 data portion.
Otherwise, return the original URL unchanged.
This is needed for providers like Ollama that expect pure base64 data
rather than full data URLs.
Args:
image_url: The image URL or data URL to process
Returns:
The base64 data if it's a data URL, otherwise the original URL
"""
if image_url.startswith("data:") and ";base64," in image_url:
return image_url.split(";base64,", 1)[1]
return image_url
def extract_images_from_message(message: AllMessageValues) -> List[str]:
"""
Extract images from a message
Extract images from a message.
For data URLs (e.g., "data:image/png;base64,iVBOR..."), only the base64
data portion is extracted. This is required for providers like Ollama
that expect pure base64 data rather than full data URLs.
"""
images = []
message_content = message.get("content")
@ -1098,7 +1124,7 @@ def extract_images_from_message(message: AllMessageValues) -> List[str]:
image_url = m.get("image_url")
if image_url:
if isinstance(image_url, str):
images.append(image_url)
images.append(_extract_base64_data(image_url))
elif isinstance(image_url, dict) and "url" in image_url:
images.append(image_url["url"])
images.append(_extract_base64_data(image_url["url"]))
return images

15
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -930,7 +930,8 @@ def create_anthropic_image_param(
# Check if the image URL is an HTTP/HTTPS URL
if image_url.startswith("http://") or image_url.startswith("https://"):
# For Bedrock invoke, always convert URLs to base64 (Bedrock invoke doesn't support URLs)
# For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
# as these providers don't support URL sources for images
if is_bedrock_invoke or image_url.startswith("http://"):
base64_url = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(
@ -1496,9 +1497,10 @@ def convert_to_gemini_tool_call_result(
content_type = content.get("type", "")
if content_type == "text":
content_str += content.get("text", "")
elif content_type == "input_image":
# Extract image for inline_data (for Computer Use screenshots)
image_url = content.get("image_url", "")
elif content_type in ("input_image", "image_url"):
# Extract image for inline_data (for Computer Use screenshots and tool results)
image_url_data = content.get("image_url", "")
image_url = image_url_data.get("url", "") if isinstance(image_url_data, dict) else image_url_data
if image_url:
# Convert image to base64 blob format for Gemini
@ -2022,9 +2024,12 @@ def anthropic_messages_pt( # noqa: PLR0915
"format": image_url_value.get("format"),
}
# Bedrock invoke models have format: invoke/...
# Vertex AI Anthropic also doesn't support URL sources for images
is_bedrock_invoke = model.lower().startswith("invoke/")
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
force_base64 = is_bedrock_invoke or is_vertex_ai
_anthropic_content_element = create_anthropic_image_param(
image_url_input, format=format, is_bedrock_invoke=is_bedrock_invoke
image_url_input, format=format, is_bedrock_invoke=force_base64
)
_content_element = add_cache_control_to_content(
anthropic_content_element=_anthropic_content_element,

16
litellm/llms/__init__.py
View file

@ -45,6 +45,7 @@ def get_cost_for_web_search_request(
return 0.0
elif custom_llm_provider == "xai":
from .xai.cost_calculator import cost_per_web_search_request
return cost_per_web_search_request(usage=usage, model_info=model_info)
else:
return None
@ -110,6 +111,21 @@ def discover_guardrail_translation_mappings() -> (
verbose_logger.error(f"Error processing {module_path}: {e}")
continue
try:
from litellm.proxy._experimental.mcp_server.guardrail_translation import (
guardrail_translation_mappings as mcp_guardrail_translation_mappings,
)
discovered_mappings.update(mcp_guardrail_translation_mappings)
verbose_logger.debug(
"Loaded MCP guardrail translation mappings: %s",
list(mcp_guardrail_translation_mappings.keys()),
)
except ImportError:
verbose_logger.debug(
"MCP guardrail translation mappings not available; skipping"
)
verbose_logger.debug(
f"Discovered {len(discovered_mappings)} guardrail translation mappings: {list(discovered_mappings.keys())}"
)

11
litellm/llms/anthropic/chat/transformation.py
View file

@ -54,7 +54,10 @@ from litellm.types.utils import (
CompletionTokensDetailsWrapper,
)
from litellm.types.utils import Message as LitellmMessage
from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse
from litellm.types.utils import (
PromptTokensDetailsWrapper,
ServerToolUse,
)
from litellm.utils import (
ModelResponse,
Usage,
@ -204,9 +207,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
) # Relevant issue: https://github.com/BerriAI/litellm/issues/7755
def get_cache_control_headers(self) -> dict:
# Anthropic no longer requires the prompt-caching beta header
# Prompt caching now works automatically when cache_control is used in messages
# Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
return {
"anthropic-version": "2023-06-01",
"anthropic-beta": "prompt-caching-2024-07-31",
}
def _map_tool_choice(
@ -1034,7 +1039,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_messages = anthropic_messages_pt(
model=model,
messages=messages,
llm_provider="anthropic",
llm_provider=self.custom_llm_provider or "anthropic",
)
except Exception as e:
raise AnthropicError(

16
litellm/llms/anthropic/common_utils.py
View file

@ -12,7 +12,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import AllAnthropicToolsValues, AnthropicMcpServerTool, ANTHROPIC_HOSTED_TOOLS
from litellm.types.llms.anthropic import (
ANTHROPIC_HOSTED_TOOLS,
AllAnthropicToolsValues,
AnthropicMcpServerTool,
)
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import TokenCountResponse
@ -273,8 +277,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
beta_header = self.get_computer_tool_beta_header(computer_tool_used)
betas.append(beta_header)
if prompt_caching_set:
betas.append("prompt-caching-2024-07-31")
# Anthropic no longer requires the prompt-caching beta header
# Prompt caching now works automatically when cache_control is used in messages
# Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
if file_id_used:
betas.append("files-api-2025-04-14")
@ -305,8 +310,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
container_with_skills_used: bool = False,
) -> dict:
betas = set()
if prompt_caching_set:
betas.add("prompt-caching-2024-07-31")
# Anthropic no longer requires the prompt-caching beta header
# Prompt caching now works automatically when cache_control is used in messages
# Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
if computer_tool_used:
beta_header = self.get_computer_tool_beta_header(computer_tool_used)
betas.add(beta_header)

7
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -48,7 +48,12 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
headers = BaseAzureLLM._base_validate_azure_environment(
headers=headers, litellm_params=litellm_params_obj
)
# Azure Anthropic uses x-api-key header (not api-key)
# Convert api-key to x-api-key if present
if "api-key" in headers and "x-api-key" not in headers:
headers["x-api-key"] = headers.pop("api-key")
# Set anthropic-version header
if "anthropic-version" not in headers:
headers["anthropic-version"] = "2023-06-01"

4
litellm/llms/gemini/google_genai/transformation.py
View file

@ -312,7 +312,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
)
request_dict = cast(dict, typed_generate_content_request)
if system_instruction is not None:
request_dict["systemInstruction"] = system_instruction
return request_dict
def transform_generate_content_response(

2
litellm/llms/ollama/completion/handler.py
View file

@ -15,7 +15,7 @@ def _prepare_ollama_embedding_payload(
) -> Dict[str, Any]:
data: Dict[str, Any] = {"model": model, "input": prompts}
special_optional_params = ["truncate", "options", "keep_alive"]
special_optional_params = ["truncate", "options", "keep_alive","dimensions"]
for k, v in optional_params.items():
if k in special_optional_params:

63
litellm/llms/openai/image_generation/cost_calculator.py Normal file
View file

@ -0,0 +1,63 @@
"""
Cost calculator for OpenAI image generation models (gpt-image-1, gpt-image-1-mini)
These models use token-based pricing instead of pixel-based pricing like DALL-E.
"""
from typing import Optional
from litellm import verbose_logger
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.utils import ImageResponse
def cost_calculator(
model: str,
image_response: ImageResponse,
custom_llm_provider: Optional[str] = None,
) -> float:
"""
Calculate cost for OpenAI gpt-image-1 and gpt-image-1-mini models.
Uses the same usage format as Responses API, so we reuse the helper
to transform to chat completion format and use generic_cost_per_token.
Args:
model: The model name (e.g., "gpt-image-1", "gpt-image-1-mini")
image_response: The ImageResponse containing usage data
custom_llm_provider: Optional provider name
Returns:
float: Total cost in USD
"""
usage = getattr(image_response, "usage", None)
if usage is None:
verbose_logger.debug(
f"No usage data available for {model}, cannot calculate token-based cost"
)
return 0.0
# Transform ImageUsage to Usage using the existing helper
# ImageUsage has the same format as ResponseAPIUsage
chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
usage
)
# Use generic_cost_per_token for cost calculation
prompt_cost, completion_cost = generic_cost_per_token(
model=model,
usage=chat_usage,
custom_llm_provider=custom_llm_provider or "openai",
)
total_cost = prompt_cost + completion_cost
verbose_logger.debug(
f"OpenAI gpt-image cost calculation for {model}: "
f"prompt_cost=${prompt_cost:.6f}, completion_cost=${completion_cost:.6f}, "
f"total=${total_cost:.6f}"
)
return total_cost

4
litellm/llms/sap/chat/transformation.py
View file

@ -203,9 +203,13 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
headers: dict,
) -> dict:
supported_params = self.get_supported_openai_params(model)
# Include extra params that passed validation (e.g., thinking_config for Gemini models via allowed_openai_params)
extra_params = [k for k in optional_params if k not in supported_params and k not in {"tools", "model_version"}]
supported_params = supported_params + extra_params
model_params = {
k: v for k, v in optional_params.items() if k in supported_params
}
model_version = optional_params.pop("model_version", "latest")
template = []
for message in messages:

13
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -383,7 +383,18 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
and isinstance(_message_content, str)
):
assistant_text = _message_content
assistant_content.append(PartType(text=assistant_text)) # type: ignore
# Check if message has thought_signatures in provider_specific_fields
provider_specific_fields = assistant_msg.get("provider_specific_fields")
thought_signatures = None
if provider_specific_fields and isinstance(provider_specific_fields, dict):
thought_signatures = provider_specific_fields.get("thought_signatures")
# If we have thought signatures, add them to the part
if thought_signatures and isinstance(thought_signatures, list) and len(thought_signatures) > 0:
# Use the first signature for the text part (Gemini expects one signature per part)
assistant_content.append(PartType(text=assistant_text, thoughtSignature=thought_signatures[0])) # type: ignore
else:
assistant_content.append(PartType(text=assistant_text)) # type: ignore
## HANDLE ASSISTANT FUNCTION CALL
if (

78
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -552,24 +552,46 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"Invalid tool={}. Use `litellm.set_verbose` or `litellm --detailed_debug` to see raw request."
)
# Only include function_declarations if there are actual functions
_tools = Tools()
# Build list of Tool objects - each Tool should contain exactly one type
# per Vertex AI API spec: "A Tool object should contain exactly one type of Tool"
_tools_list: List[Tools] = []
# Function declarations can be grouped together in one Tool
if gtool_func_declarations:
_tools["function_declarations"] = gtool_func_declarations
func_tool = Tools()
func_tool["function_declarations"] = gtool_func_declarations
_tools_list.append(func_tool)
# Each special tool type must be in its own Tool object
if googleSearch is not None:
_tools[VertexToolName.GOOGLE_SEARCH.value] = googleSearch
search_tool = Tools()
search_tool[VertexToolName.GOOGLE_SEARCH.value] = googleSearch
_tools_list.append(search_tool)
if googleSearchRetrieval is not None:
_tools[VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value] = googleSearchRetrieval
retrieval_tool = Tools()
retrieval_tool[VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value] = googleSearchRetrieval
_tools_list.append(retrieval_tool)
if enterpriseWebSearch is not None:
_tools[VertexToolName.ENTERPRISE_WEB_SEARCH.value] = enterpriseWebSearch
enterprise_tool = Tools()
enterprise_tool[VertexToolName.ENTERPRISE_WEB_SEARCH.value] = enterpriseWebSearch
_tools_list.append(enterprise_tool)
if code_execution is not None:
_tools[VertexToolName.CODE_EXECUTION.value] = code_execution
code_tool = Tools()
code_tool[VertexToolName.CODE_EXECUTION.value] = code_execution
_tools_list.append(code_tool)
if urlContext is not None:
_tools[VertexToolName.URL_CONTEXT.value] = urlContext
url_tool = Tools()
url_tool[VertexToolName.URL_CONTEXT.value] = urlContext
_tools_list.append(url_tool)
if googleMaps is not None:
_tools[VertexToolName.GOOGLE_MAPS.value] = googleMaps
maps_tool = Tools()
maps_tool[VertexToolName.GOOGLE_MAPS.value] = googleMaps
_tools_list.append(maps_tool)
if computerUse is not None:
_tools[VertexToolName.COMPUTER_USE.value] = computerUse
computer_tool = Tools()
computer_tool[VertexToolName.COMPUTER_USE.value] = computerUse
_tools_list.append(computer_tool)
# Add retrieval config to toolConfig if googleMaps has location data
if google_maps_retrieval_config is not None:
@ -579,7 +601,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"retrievalConfig"
] = google_maps_retrieval_config
return [_tools]
return _tools_list
def _map_response_schema(self, value: dict) -> dict:
old_schema = deepcopy(value)
@ -1210,6 +1232,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
thinking_blocks.append(block)
return thinking_blocks
def _extract_thought_signatures_from_parts(
self, parts: List[HttpxPartType]
) -> Optional[List[str]]:
"""Extract thoughtSignature values from parts.
Per Google's docs, thoughtSignature is returned for multi-turn context preservation
and can appear on parts even without thought: true (e.g., regular text responses,
function calls). This method extracts all thoughtSignature values from parts.
Returns:
List of thoughtSignature strings if any are found, None otherwise
"""
signatures: List[str] = []
for part in parts:
signature = part.get("thoughtSignature")
if signature is not None:
signatures.append(signature)
return signatures if signatures else None
def _extract_image_response_from_parts(
self, parts: List[HttpxPartType]
) -> Optional[List[ImageURLListItem]]:
@ -1620,6 +1661,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
from litellm.types.utils import Delta, StreamingChoices
annotations = chat_completion_message.get("annotations") # type: ignore
provider_specific_fields = chat_completion_message.get("provider_specific_fields") # type: ignore
# create a streaming choice object
choice = StreamingChoices(
finish_reason=VertexGeminiConfig._check_finish_reason(
@ -1633,6 +1675,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
images=image_response,
function_call=functions,
annotations=annotations, # type: ignore
provider_specific_fields=provider_specific_fields,
),
logprobs=chat_completion_logprobs,
enhancements=None,
@ -1811,6 +1854,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
)
)
# Extract thoughtSignatures from parts (can exist without thought: true)
thought_signatures = (
VertexGeminiConfig()._extract_thought_signatures_from_parts(
parts=candidate["content"]["parts"]
)
)
if audio_response is not None:
cast(Dict[str, Any], chat_completion_message)[
"audio"
@ -1876,6 +1926,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
reasoning_content = "\n".join(reasoning_content_parts)
chat_completion_message["reasoning_content"] = reasoning_content
# Store thoughtSignatures in provider_specific_fields
if thought_signatures is not None:
if "provider_specific_fields" not in chat_completion_message:
chat_completion_message["provider_specific_fields"] = {}
chat_completion_message["provider_specific_fields"]["thought_signatures"] = thought_signatures # type: ignore
if isinstance(model_response, ModelResponseStream):
choice = VertexGeminiConfig._create_streaming_choice(
chat_completion_message=chat_completion_message,

11
litellm/llms/zai/chat/transformation.py
View file

@ -20,7 +20,7 @@ class ZAIChatConfig(OpenAIGPTConfig):
return api_base, dynamic_api_key
def get_supported_openai_params(self, model: str) -> list:
return [
base_params = [
"max_tokens",
"stream",
"stream_options",
@ -31,3 +31,12 @@ class ZAIChatConfig(OpenAIGPTConfig):
"tool_choice",
]
import litellm
try:
if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider):
base_params.append("thinking")
except Exception:
pass
return base_params

452
litellm/model_prices_and_context_window_backup.json
View file

@ -249,6 +249,30 @@
"/v1/images/generations"
]
},
"aiml/google/imagen-4.0-ultra-generate-001": {
"litellm_provider": "aiml",
"metadata": {
"notes": "Imagen 4.0 Ultra Generate API - Photorealistic image generation with precise text rendering"
},
"mode": "image_generation",
"output_cost_per_image": 0.063,
"source": "https://docs.aimlapi.com/api-references/image-models/google/imagen-4-ultra-generate",
"supported_endpoints": [
"/v1/images/generations"
]
},
"aiml/google/nano-banana-pro": {
"litellm_provider": "aiml",
"metadata": {
"notes": "Gemini 3 Pro Image (Nano Banana Pro) - Advanced text-to-image generation with reasoning and 4K resolution support"
},
"mode": "image_generation",
"output_cost_per_image": 0.1575,
"source": "https://docs.aimlapi.com/api-references/image-models/google/gemini-3-pro-image-preview",
"supported_endpoints": [
"/v1/images/generations"
]
},
"amazon.nova-canvas-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 2600,
@ -3508,6 +3532,40 @@
"supports_service_tier": true,
"supports_vision": true
},
"azure/gpt-5.2-chat": {
"cache_read_input_token_cost": 1.75e-07,
"cache_read_input_token_cost_priority": 3.5e-07,
"input_cost_per_token": 1.75e-06,
"input_cost_per_token_priority": 3.5e-06,
"litellm_provider": "azure",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 1.4e-05,
"output_cost_per_token_priority": 2.8e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/gpt-5.2-chat-2025-12-11": {
"cache_read_input_token_cost": 1.75e-07,
"cache_read_input_token_cost_priority": 3.5e-07,
@ -3605,12 +3663,16 @@
"supports_web_search": true
},
"azure/gpt-image-1": {
"input_cost_per_pixel": 4.0054321e-08,
"cache_read_input_image_token_cost": 2.5e-06,
"cache_read_input_token_cost": 1.25e-06,
"input_cost_per_image_token": 1e-05,
"input_cost_per_token": 5e-06,
"litellm_provider": "azure",
"mode": "image_generation",
"output_cost_per_pixel": 0.0,
"output_cost_per_image_token": 4e-05,
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"azure/hd/1024-x-1024/dall-e-3": {
@ -3713,12 +3775,16 @@
]
},
"azure/gpt-image-1-mini": {
"input_cost_per_pixel": 8.0566406e-09,
"cache_read_input_image_token_cost": 2.5e-07,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_image_token": 2.5e-06,
"input_cost_per_token": 2e-06,
"litellm_provider": "azure",
"mode": "image_generation",
"output_cost_per_pixel": 0.0,
"output_cost_per_image_token": 8e-06,
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"azure/gpt-image-1.5": {
@ -10885,13 +10951,13 @@
"supports_tool_choice": true
},
"fireworks_ai/accounts/fireworks/models/deepseek-v3p2": {
"input_cost_per_token": 1.2e-06,
"input_cost_per_token": 5.6e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"output_cost_per_token": 1.68e-06,
"source": "https://fireworks.ai/models/fireworks/deepseek-v3p2",
"supports_function_calling": true,
"supports_reasoning": true,
@ -16922,6 +16988,336 @@
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.034,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.133,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.034,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.133,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"gpt-5": {
"cache_read_input_token_cost": 1.25e-07,
"cache_read_input_token_cost_flex": 6.25e-08,
@ -17683,16 +18079,16 @@
"supports_vision": true
},
"gpt-image-1": {
"input_cost_per_image": 0.042,
"input_cost_per_pixel": 4.0054321e-08,
"input_cost_per_token": 0.000005,
"input_cost_per_image_token": 0.00001,
"cache_read_input_image_token_cost": 2.5e-06,
"cache_read_input_token_cost": 1.25e-06,
"input_cost_per_image_token": 1e-05,
"input_cost_per_token": 5e-06,
"litellm_provider": "openai",
"mode": "image_generation",
"output_cost_per_pixel": 0.0,
"output_cost_per_token": 0.00004,
"output_cost_per_image_token": 4e-05,
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"gpt-image-1-mini": {
@ -18117,6 +18513,18 @@
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/gemma-7b-it": {
"input_cost_per_token": 5e-08,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 8e-08,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/meta-llama/llama-guard-4-12b": {
"input_cost_per_token": 2e-07,
"litellm_provider": "groq",
@ -29241,6 +29649,20 @@
"supports_vision": true,
"supports_web_search": true
},
"zai/glm-4.7": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 1.1e-07,
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.2e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-4.6": {
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.2e-06,

16
litellm/proxy/_experimental/mcp_server/guardrail_translation/__init__.py Normal file
View file

@ -0,0 +1,16 @@
"""Guardrail translation mapping for MCP tool calls."""
from litellm.proxy._experimental.mcp_server.guardrail_translation.handler import (
MCPGuardrailTranslationHandler,
)
from litellm.types.utils import CallTypes
# This mapping lives alongside the MCP server implementation because MCP
# integrations are managed by the proxy subsystem, not litellm.llms providers.
# Unified guardrails import this module explicitly to register the handler.
guardrail_translation_mappings = {
CallTypes.call_mcp_tool: MCPGuardrailTranslationHandler,
}
__all__ = ["guardrail_translation_mappings", "MCPGuardrailTranslationHandler"]

89
litellm/proxy/_experimental/mcp_server/guardrail_translation/handler.py Normal file
View file

@ -0,0 +1,89 @@
"""
MCP Guardrail Handler for Unified Guardrails.
This handler works with the synthetic "messages" payload generated by
`ProxyLogging._convert_mcp_to_llm_format`, which always produces a single user
message whose `content` string encodes the MCP tool name and arguments. The
handler simply feeds that text through the configured guardrail and writes the
result back onto the message.
"""
from typing import TYPE_CHECKING, Any, Dict, Optional
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.types.utils import GenericGuardrailAPIInputs
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from mcp.types import CallToolResult
class MCPGuardrailTranslationHandler(BaseTranslation):
"""Guardrail translation handler for MCP tool calls."""
async def process_input_messages(
self,
data: Dict[str, Any],
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional[Any] = None,
) -> Dict[str, Any]:
messages = data.get("messages")
if not isinstance(messages, list) or not messages:
verbose_proxy_logger.debug("MCP Guardrail: No messages to process")
return data
first_message = messages[0]
content: Optional[str] = None
if isinstance(first_message, dict):
content = first_message.get("content")
else:
content = getattr(first_message, "content", None)
if not isinstance(content, str):
verbose_proxy_logger.debug(
"MCP Guardrail: Message content missing or not a string",
)
return data
guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
inputs=GenericGuardrailAPIInputs(texts=[content]),
request_data=data,
input_type="request",
logging_obj=litellm_logging_obj,
)
guardrailed_texts = (
guardrailed_inputs.get("texts", []) if guardrailed_inputs else []
)
if guardrailed_texts:
new_content = guardrailed_texts[0]
if isinstance(first_message, dict):
first_message["content"] = new_content
else:
setattr(first_message, "content", new_content)
verbose_proxy_logger.debug(
"MCP Guardrail: Updated content for tool %s",
data.get("mcp_tool_name"),
)
else:
verbose_proxy_logger.debug(
"MCP Guardrail: Guardrail returned no text updates for tool %s",
data.get("mcp_tool_name"),
)
return data
async def process_output_response(
self,
response: "CallToolResult",
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional[Any] = None,
user_api_key_dict: Optional[Any] = None,
) -> Any:
# Not implemented: MCP guardrail translation never calls this path today.
verbose_proxy_logger.debug(
"MCP Guardrail: Output processing not implemented for MCP tools",
)
return response

349
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -11,7 +11,7 @@ import datetime
import hashlib
import json
import re
from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast
from urllib.parse import urlparse
from fastapi import HTTPException
@ -30,6 +30,7 @@ from pydantic import AnyUrl
import litellm
from litellm._logging import verbose_logger
from litellm.types.utils import CallTypes
from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from litellm.experimental_mcp_client.client import MCPClient
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
@ -536,9 +537,12 @@ class MCPServerManager:
client_secret=client_secret_value
or getattr(mcp_server, "client_secret", None),
scopes=resolved_scopes,
authorization_url=getattr(mcp_oauth_metadata, "authorization_url", None),
token_url=getattr(mcp_oauth_metadata, "token_url", None),
registration_url=getattr(mcp_oauth_metadata, "registration_url", None),
authorization_url=mcp_server.authorization_url
or getattr(mcp_oauth_metadata, "authorization_url", None),
token_url=mcp_server.token_url
or getattr(mcp_oauth_metadata, "token_url", None),
registration_url=mcp_server.registration_url
or getattr(mcp_oauth_metadata, "registration_url", None),
command=getattr(mcp_server, "command", None),
args=getattr(mcp_server, "args", None) or [],
env=env_dict,
@ -548,7 +552,7 @@ class MCPServerManager:
)
return new_server
async def add_update_server(self, mcp_server: LiteLLM_MCPServerTable):
async def add_server(self, mcp_server: LiteLLM_MCPServerTable):
try:
if mcp_server.server_id not in self.registry:
new_server = await self.build_mcp_server_from_table(mcp_server)
@ -559,6 +563,17 @@ class MCPServerManager:
verbose_logger.debug(f"Failed to add MCP server: {str(e)}")
raise e
async def update_server(self, mcp_server: LiteLLM_MCPServerTable):
try:
if mcp_server.server_id in self.registry:
new_server = await self.build_mcp_server_from_table(mcp_server)
self.registry[mcp_server.server_id] = new_server
verbose_logger.debug(f"Updated MCP Server: {new_server.name}")
except Exception as e:
verbose_logger.debug(f"Failed to udpate MCP server: {str(e)}")
raise e
def get_all_mcp_server_ids(self) -> Set[str]:
"""
Get all MCP server IDs
@ -1662,11 +1677,11 @@ class MCPServerManager:
)
try:
# Use standard pre_call_hook with call_type="mcp_call"
# Use standard pre_call_hook
modified_data = await proxy_logging_obj.pre_call_hook(
user_api_key_dict=user_api_key_auth, # type: ignore
data=synthetic_llm_data,
call_type="mcp_call", # type: ignore
call_type=CallTypes.call_mcp_tool.value,
)
if modified_data:
# Convert response back to MCP format and apply modifications
@ -1723,7 +1738,7 @@ class MCPServerManager:
proxy_logging_obj.during_call_hook(
user_api_key_dict=user_api_key_auth,
data=synthetic_llm_data,
call_type="mcp_call", # type: ignore
call_type=CallTypes.call_mcp_tool.value,
)
)
@ -1879,7 +1894,7 @@ class MCPServerManager:
#########################################################
# Pre MCP Tool Call Hook
# Allow validation and modification of tool calls before execution
# Using standard pre_call_hook with call_type="mcp_call"
# Using standard pre_call_hook
#########################################################
if proxy_logging_obj:
await self.pre_call_tool_check(
@ -2040,7 +2055,7 @@ class MCPServerManager:
verbose_logger.debug(
f"Adding server to registry: {server.server_id} ({server.server_name})"
)
await self.add_update_server(server)
await self.add_server(server)
verbose_logger.debug(
f"Registry now contains {len(self.get_registry())} servers"
@ -2127,7 +2142,7 @@ class MCPServerManager:
async def health_check_server(
self, server_id: str, mcp_auth_header: Optional[str] = None
) -> Dict[str, Any]:
) -> LiteLLM_MCPServerTable:
"""
Perform a health check on a specific MCP server.
@ -2138,206 +2153,186 @@ class MCPServerManager:
Returns:
Dict containing health check results
"""
import time
from datetime import datetime
server = self.get_mcp_server_by_id(server_id)
if not server:
return {
"server_id": server_id,
"server_name": None,
"status": "unknown",
"error": "Server not found",
"last_health_check": datetime.now().isoformat(),
"response_time_ms": None,
}
start_time = time.time()
try:
# Try to get tools from the server as a health check
tools = await self._get_tools_from_server(server, mcp_auth_header)
response_time = (time.time() - start_time) * 1000
return {
"server_id": server_id,
"server_name": server.name,
"status": "healthy",
"tools_count": len(tools),
"last_health_check": datetime.now().isoformat(),
"response_time_ms": round(response_time, 2),
"error": None,
}
except Exception as e:
response_time = (time.time() - start_time) * 1000
error_message = str(e)
return {
"server_id": server_id,
"server_name": server.name,
"status": "unhealthy",
"last_health_check": datetime.now().isoformat(),
"response_time_ms": round(response_time, 2),
"error": error_message,
}
async def health_check_all_servers(
self, mcp_auth_header: Optional[str] = None
) -> Dict[str, Any]:
"""
Perform health checks on all MCP servers.
Args:
mcp_auth_header: Optional authentication header for the MCP servers
Returns:
Dict containing health check results for all servers
"""
all_servers = self.get_registry()
results = {}
for server_id, server in all_servers.items():
results[server_id] = await self.health_check_server(
server_id, mcp_auth_header
verbose_logger.warning(f"MCP Server {server_id} not found")
return LiteLLM_MCPServerTable(
server_id=server_id,
server_name=None,
transport=MCPTransport.http, # Default transport for not found servers
status="unknown",
health_check_error="Server not found",
last_health_check=datetime.now(),
)
return results
status: Literal["healthy", "unhealthy", "unknown"] = "unknown"
health_check_error = None
async def health_check_allowed_servers(
self,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
) -> Dict[str, Any]:
"""
Perform health checks on all MCP servers that the user has access to.
# Check if we should skip health check based on auth configuration
should_skip_health_check = False
Args:
user_api_key_auth: User authentication info for access control
mcp_auth_header: Optional authentication header for the MCP servers
# Skip if auth_type is oauth2
if server.auth_type == MCPAuth.oauth2:
should_skip_health_check = True
# Skip if auth_type is not none and authentication_token is missing
elif (
server.auth_type
and server.auth_type != MCPAuth.none
and not server.authentication_token
):
should_skip_health_check = True
Returns:
Dict containing health check results for accessible servers
"""
# Get allowed servers for the user
allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth)
if not should_skip_health_check:
extra_headers = {}
if server.static_headers:
extra_headers.update(server.static_headers)
# Perform health checks on allowed servers
results = {}
for server_id in allowed_server_ids:
results[server_id] = await self.health_check_server(
server_id, mcp_auth_header
client = self._create_mcp_client(
server=server,
mcp_auth_header=None,
extra_headers=extra_headers,
stdio_env=None,
)
return results
try:
async def _noop(session):
return "ok"
# Add timeout wrapper to prevent hanging
await asyncio.wait_for(client.run_with_session(_noop), timeout=10.0)
status = "healthy"
except asyncio.TimeoutError:
health_check_error = "Health check timed out after 10 seconds"
status = "unhealthy"
except Exception as e:
health_check_error = str(e)
status = "unhealthy"
return LiteLLM_MCPServerTable(
server_id=server.server_id,
server_name=server.server_name,
alias=server.alias,
description=(
server.mcp_info.get("description") if server.mcp_info else None
),
url=server.url,
transport=server.transport,
auth_type=server.auth_type,
created_at=datetime.now(),
updated_at=datetime.now(),
teams=[],
mcp_access_groups=server.access_groups or [],
allowed_tools=server.allowed_tools or [],
extra_headers=server.extra_headers or [],
mcp_info=server.mcp_info,
static_headers=server.static_headers,
status=status,
last_health_check=datetime.now(),
health_check_error=health_check_error,
command=getattr(server, "command", None),
args=getattr(server, "args", None) or [],
env=getattr(server, "env", None) or {},
authorization_url=server.authorization_url,
token_url=server.token_url,
registration_url=server.registration_url,
)
async def get_all_mcp_servers_with_health_and_teams(
self,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
include_health: bool = True,
server_ids: Optional[List[str]] = None,
) -> List[LiteLLM_MCPServerTable]:
"""
Get all MCP servers that the user has access to, with health status and team information.
Args:
user_api_key_auth: User authentication info for access control
include_health: Whether to include health check information
server_ids: Optional list of server IDs to filter. If provided, only these servers
will be checked (subject to access control). If None, all accessible servers are checked.
Returns:
List of MCP server objects with health and team data
"""
from litellm.proxy._experimental.mcp_server.db import (
get_all_mcp_servers,
get_mcp_servers,
)
from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
from litellm.proxy.proxy_server import prisma_client
# Get allowed server IDs
allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth)
# Get servers from database
# Filter by requested server_ids if provided
if server_ids:
# Only check servers that are both requested AND accessible
target_server_ids = [sid for sid in server_ids if sid in allowed_server_ids]
else:
# Check all accessible servers
target_server_ids = allowed_server_ids
# Run health checks concurrently
tasks = [self.health_check_server(server_id) for server_id in target_server_ids]
results = await asyncio.gather(*tasks)
# Filter out None results (servers that were not found)
list_mcp_servers = [server for server in results if server is not None]
return list_mcp_servers
async def get_all_allowed_mcp_servers(
self,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
) -> List[LiteLLM_MCPServerTable]:
"""
Get all MCP servers that the user has access to.
Args:
user_api_key_auth: User authentication info for access control
Returns:
List of MCP server objects without health status
"""
from datetime import datetime
# Get allowed server IDs
allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth)
list_mcp_servers: List[LiteLLM_MCPServerTable] = []
if prisma_client is not None:
list_mcp_servers = await get_mcp_servers(prisma_client, allowed_server_ids)
# If admin, also get all servers from database
if user_api_key_auth and _user_has_admin_view(user_api_key_auth):
all_mcp_servers = await get_all_mcp_servers(prisma_client)
for server in all_mcp_servers:
if server.server_id not in allowed_server_ids:
list_mcp_servers.append(server)
for server_id in allowed_server_ids:
server = self.get_mcp_server_by_id(server_id)
if not server:
verbose_logger.warning(f"MCP Server {server_id} not found in registry")
continue
# Add config.yaml servers
for _server_id, _server_config in self.config_mcp_servers.items():
if _server_id in allowed_server_ids:
list_mcp_servers.append(
LiteLLM_MCPServerTable(
**{
**_server_config.model_dump(),
"created_at": datetime.datetime.now(),
"updated_at": datetime.datetime.now(),
"description": (
_server_config.mcp_info.get("description")
if _server_config.mcp_info
else None
),
"allowed_tools": _server_config.allowed_tools or [],
"mcp_info": _server_config.mcp_info,
"mcp_access_groups": _server_config.access_groups or [],
"extra_headers": _server_config.extra_headers or [],
"command": getattr(_server_config, "command", None),
"args": getattr(_server_config, "args", None) or [],
"env": getattr(_server_config, "env", None) or {},
}
)
)
# Get team information for non-admin users
server_to_teams_map: Dict[str, List[Dict[str, str]]] = {}
if (
user_api_key_auth
and not _user_has_admin_view(user_api_key_auth)
and prisma_client is not None
):
teams = await prisma_client.db.litellm_teamtable.find_many(
include={"object_permission": True}
# Build LiteLLM_MCPServerTable without health check
mcp_server_table = LiteLLM_MCPServerTable(
server_id=server.server_id,
server_name=server.server_name,
alias=server.alias,
description=(
server.mcp_info.get("description") if server.mcp_info else None
),
url=server.url,
transport=server.transport,
auth_type=server.auth_type,
created_at=datetime.now(),
updated_at=datetime.now(),
teams=[],
mcp_access_groups=server.access_groups or [],
allowed_tools=server.allowed_tools or [],
extra_headers=server.extra_headers or [],
mcp_info=server.mcp_info,
static_headers=server.static_headers,
status=None, # No health check performed
last_health_check=None, # No health check performed
health_check_error=None,
command=getattr(server, "command", None),
args=getattr(server, "args", None) or [],
env=getattr(server, "env", None) or {},
authorization_url=server.authorization_url,
token_url=server.token_url,
registration_url=server.registration_url,
)
user_teams = []
for team in teams:
if team.members_with_roles:
for member in team.members_with_roles:
if (
"user_id" in member
and member["user_id"] is not None
and member["user_id"] == user_api_key_auth.user_id
):
user_teams.append(team)
# Create a mapping of server_id to teams that have access to it
for team in user_teams:
if team.object_permission and team.object_permission.mcp_servers:
for server_id in team.object_permission.mcp_servers:
if server_id not in server_to_teams_map:
server_to_teams_map[server_id] = []
server_to_teams_map[server_id].append(
{
"team_id": team.team_id,
"team_alias": team.team_alias,
"organization_id": team.organization_id,
}
)
## mark invalid servers w/ reason for being invalid
valid_server_ids = self.get_all_mcp_server_ids()
for server in list_mcp_servers:
if server.server_id not in valid_server_ids:
server.status = "unhealthy"
## try adding server to registry to get error
try:
await self.add_update_server(server)
except Exception as e:
server.health_check_error = str(e)
server.health_check_error = "Server is not in in memory registry yet. This could be a temporary sync issue."
list_mcp_servers.append(mcp_server_table)
return list_mcp_servers

35
litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py
View file

@ -3,7 +3,9 @@ This module is used to generate MCP tools from OpenAPI specs.
"""
import json
from pathlib import PurePosixPath
from typing import Any, Dict, Optional
from urllib.parse import quote
import httpx
@ -17,6 +19,29 @@ BASE_URL = ""
HEADERS: Dict[str, str] = {}
def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str:
"""Ensure path params cannot introduce directory traversal."""
if param_value is None:
return ""
value_str = str(param_value)
if value_str == "":
return ""
normalized_value = value_str.replace("\\", "/")
if "/" in normalized_value:
raise ValueError(
f"Path parameter '{param_name}' must not contain path separators"
)
if any(part in {".", ".."} for part in PurePosixPath(normalized_value).parts):
raise ValueError(
f"Path parameter '{param_name}' cannot include '.' or '..' segments"
)
return quote(value_str, safe="")
def load_openapi_spec(filepath: str) -> Dict[str, Any]:
"""Load OpenAPI specification from JSON file."""
with open(filepath, "r") as f:
@ -144,12 +169,18 @@ def create_tool_function(
url = base_url + path
# Replace path parameters using original names from OpenAPI spec
# Apply path traversal validation and URL encoding
for param_name in path_params:
param_value = kwargs.get(param_name, "")
if param_value:
try:
# Sanitize and encode path parameter to prevent traversal attacks
safe_value = _sanitize_path_parameter_value(param_value, param_name)
except ValueError as exc:
return "Invalid path parameter: " + str(exc)
# Replace {param_name} or {{param_name}} in URL
url = url.replace("{" + param_name + "}", str(param_value))
url = url.replace("{{" + param_name + "}}", str(param_value))
url = url.replace("{" + param_name + "}", safe_value)
url = url.replace("{{" + param_name + "}}", safe_value)
# Build query params using original parameter names
params: Dict[str, Any] = {}

8
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -1,5 +1,4 @@
import importlib
import traceback
from typing import Dict, List, Optional, Union
from fastapi import APIRouter, Depends, Query, Request
@ -347,17 +346,16 @@ if MCP_AVAILABLE:
except Exception as e:
verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True)
stack_trace = traceback.format_exc()
return {
"status": "error",
"message": f"An internal error has occurred: {str(e)}",
"stack_trace": stack_trace,
"message": "An internal error has occurred while testing the MCP server.",
}
@router.post("/test/connection")
@router.post("/test/connection", dependencies=[Depends(user_api_key_auth)])
async def test_connection(
request: Request,
new_mcp_server_request: NewMCPServerRequest,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Test if we can connect to the provided MCP server before adding it

124
litellm/proxy/_types.py
View file

@ -412,7 +412,6 @@ class LiteLLMRoutes(enum.Enum):
agent_routes = [
"/v1/agents",
"/agents",
"/a2a/{agent_id}",
"/a2a/{agent_id}/message/send",
"/a2a/{agent_id}/message/stream",
@ -830,9 +829,9 @@ class GenerateRequestBase(LiteLLMPydanticObjectBase):
allowed_cache_controls: Optional[list] = []
config: Optional[dict] = {}
permissions: Optional[dict] = {}
model_max_budget: Optional[dict] = (
{}
) # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {}
model_max_budget: Optional[
dict
] = {} # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {}
model_config = ConfigDict(protected_namespaces=())
model_rpm_limit: Optional[dict] = None
@ -1035,6 +1034,9 @@ class NewMCPServerRequest(LiteLLMPydanticObjectBase):
command: Optional[str] = None
args: List[str] = Field(default_factory=list)
env: Dict[str, str] = Field(default_factory=dict)
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
@model_validator(mode="before")
@classmethod
@ -1092,6 +1094,9 @@ class UpdateMCPServerRequest(LiteLLMPydanticObjectBase):
command: Optional[str] = None
args: List[str] = Field(default_factory=list)
env: Dict[str, str] = Field(default_factory=dict)
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
@model_validator(mode="before")
@classmethod
@ -1141,6 +1146,9 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
command: Optional[str] = None
args: List[str] = Field(default_factory=list)
env: Dict[str, str] = Field(default_factory=dict)
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
class MakeMCPServersPublicRequest(LiteLLMPydanticObjectBase):
@ -1160,6 +1168,9 @@ class NewSkillRequest(LiteLLMPydanticObjectBase):
file_name: Optional[str] = None # Original filename
file_type: Optional[str] = None # MIME type (e.g., "application/zip")
metadata: Optional[Dict[str, Any]] = None
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
class UpdateSkillRequest(LiteLLMPydanticObjectBase):
@ -1347,12 +1358,12 @@ class NewCustomerRequest(BudgetNewRequest):
blocked: bool = False # allow/disallow requests for this end-user
budget_id: Optional[str] = None # give either a budget_id or max_budget
spend: Optional[float] = None
allowed_model_region: Optional[AllowedModelRegion] = (
None # require all user requests to use models in this specific region
)
default_model: Optional[str] = (
None # if no equivalent model in allowed region - default all requests to this model
)
allowed_model_region: Optional[
AllowedModelRegion
] = None # require all user requests to use models in this specific region
default_model: Optional[
str
] = None # if no equivalent model in allowed region - default all requests to this model
@model_validator(mode="before")
@classmethod
@ -1374,12 +1385,12 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase):
blocked: bool = False # allow/disallow requests for this end-user
max_budget: Optional[float] = None
budget_id: Optional[str] = None # give either a budget_id or max_budget
allowed_model_region: Optional[AllowedModelRegion] = (
None # require all user requests to use models in this specific region
)
default_model: Optional[str] = (
None # if no equivalent model in allowed region - default all requests to this model
)
allowed_model_region: Optional[
AllowedModelRegion
] = None # require all user requests to use models in this specific region
default_model: Optional[
str
] = None # if no equivalent model in allowed region - default all requests to this model
class DeleteCustomerRequest(LiteLLMPydanticObjectBase):
@ -1464,15 +1475,15 @@ class NewTeamRequest(TeamBase):
] = None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm
model_tpm_limit: Optional[Dict[str, int]] = None
team_member_budget: Optional[float] = (
None # allow user to set a budget for all team members
)
team_member_rpm_limit: Optional[int] = (
None # allow user to set RPM limit for all team members
)
team_member_tpm_limit: Optional[int] = (
None # allow user to set TPM limit for all team members
)
team_member_budget: Optional[
float
] = None # allow user to set a budget for all team members
team_member_rpm_limit: Optional[
int
] = None # allow user to set RPM limit for all team members
team_member_tpm_limit: Optional[
int
] = None # allow user to set TPM limit for all team members
team_member_key_duration: Optional[str] = None # e.g. "1d", "1w", "1m"
allowed_vector_store_indexes: Optional[List[AllowedVectorStoreIndexItem]] = None
@ -1558,9 +1569,9 @@ class BlockKeyRequest(LiteLLMPydanticObjectBase):
class AddTeamCallback(LiteLLMPydanticObjectBase):
callback_name: str
callback_type: Optional[Literal["success", "failure", "success_and_failure"]] = (
"success_and_failure"
)
callback_type: Optional[
Literal["success", "failure", "success_and_failure"]
] = "success_and_failure"
callback_vars: Dict[str, str]
@model_validator(mode="before")
@ -1788,9 +1799,10 @@ class DynamoDBArgs(LiteLLMPydanticObjectBase):
class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase):
"""
Settings for a specific guardrail on a passthrough endpoint.
Allows field-level targeting for guardrail execution.
"""
request_fields: Optional[List[str]] = Field(
default=None,
description="JSONPath expressions for input field targeting (pre_call). Examples: 'query', 'documents[*].text', 'messages[*].content'. If not specified, guardrail runs on entire request payload.",
@ -1871,9 +1883,9 @@ class ConfigList(LiteLLMPydanticObjectBase):
stored_in_db: Optional[bool]
field_default_value: Any
premium_field: bool = False
nested_fields: Optional[List[FieldDetail]] = (
None # For nested dictionary or Pydantic fields
)
nested_fields: Optional[
List[FieldDetail]
] = None # For nested dictionary or Pydantic fields
class UserHeaderMapping(LiteLLMPydanticObjectBase):
@ -2152,6 +2164,7 @@ class UserAPIKeyAuth(
user_rpm_limit: Optional[int] = None
user_email: Optional[str] = None
request_route: Optional[str] = None
user: Optional[Any] = None # Expanded user object when expand=user is used
model_config = ConfigDict(arbitrary_types_allowed=True)
@ -2258,9 +2271,9 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase):
budget_id: Optional[str] = None
created_at: datetime
updated_at: datetime
user: Optional[Any] = (
None # You might want to replace 'Any' with a more specific type if available
)
user: Optional[
Any
] = None # You might want to replace 'Any' with a more specific type if available
litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
model_config = ConfigDict(protected_namespaces=())
@ -2702,7 +2715,7 @@ class AllCallbacks(LiteLLMPydanticObjectBase):
"TRACELOOP_API_KEY",
],
ui_callback_name="Traceloop",
)
)
class SpendLogsMetadata(TypedDict):
@ -2736,9 +2749,7 @@ class SpendLogsMetadata(TypedDict):
cold_storage_object_key: Optional[
str
] # S3/GCS object key for cold storage retrieval
litellm_overhead_time_ms: Optional[
float
] # LiteLLM overhead time in milliseconds
litellm_overhead_time_ms: Optional[float] # LiteLLM overhead time in milliseconds
cost_breakdown: Optional[
CostBreakdown
] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
@ -3222,9 +3233,9 @@ class TeamModelDeleteRequest(BaseModel):
# Organization Member Requests
class OrganizationMemberAddRequest(OrgMemberAddRequest):
organization_id: str
max_budget_in_organization: Optional[float] = (
None # Users max budget within the organization
)
max_budget_in_organization: Optional[
float
] = None # Users max budget within the organization
class OrganizationMemberDeleteRequest(MemberDeleteRequest):
@ -3439,9 +3450,9 @@ class ProviderBudgetResponse(LiteLLMPydanticObjectBase):
Maps provider names to their budget configs.
"""
providers: Dict[str, ProviderBudgetResponseObject] = (
{}
) # Dictionary mapping provider names to their budget configurations
providers: Dict[
str, ProviderBudgetResponseObject
] = {} # Dictionary mapping provider names to their budget configurations
class ProxyStateVariables(TypedDict):
@ -3560,8 +3571,16 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
default=None,
description="If no team_id given, default permissions/spend-tracking to this team.s",
)
team_alias_jwt_field: Optional[str] = Field(
default=None,
description="The field in the JWT token that stores the team name/alias. Will be resolved to team_id via database lookup.",
)
org_id_jwt_field: Optional[str] = None
org_alias_jwt_field: Optional[str] = Field(
default=None,
description="The field in the JWT token that stores the organization name/alias. Will be resolved to org_id via database lookup.",
)
user_id_jwt_field: Optional[str] = None
user_email_jwt_field: Optional[str] = None
user_allowed_email_domain: Optional[str] = None
@ -3576,9 +3595,9 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
enforce_rbac: bool = False
roles_jwt_field: Optional[str] = None # v2 on role mappings
role_mappings: Optional[List[RoleMapping]] = None
object_id_jwt_field: Optional[str] = (
None # can be either user / team, inferred from the role mapping
)
object_id_jwt_field: Optional[
str
] = None # can be either user / team, inferred from the role mapping
scope_mappings: Optional[List[ScopeMapping]] = None
enforce_scope_based_access: bool = False
enforce_team_based_model_access: bool = False
@ -3729,13 +3748,16 @@ class DailyOrganizationSpendTransaction(BaseDailySpendTransaction):
class DailyUserSpendTransaction(BaseDailySpendTransaction):
user_id: str
class DailyEndUserSpendTransaction(BaseDailySpendTransaction):
end_user_id: str
class DailyTagSpendTransaction(BaseDailySpendTransaction):
request_id: Optional[str]
tag: str
class DailyAgentSpendTransaction(BaseDailySpendTransaction):
agent_id: str
@ -3767,8 +3789,8 @@ class LiteLLM_ManagedFileTable(LiteLLMPydanticObjectBase):
flat_model_file_ids: List[str]
created_by: Optional[str]
updated_by: Optional[str]
storage_backend: Optional[str] = None
storage_url: Optional[str] = None
storage_backend: Optional[str] = None
storage_url: Optional[str] = None
class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase):
@ -3799,4 +3821,4 @@ class LiteLLM_ManagedVectorStoresTable(LiteLLMPydanticObjectBase):
class ResponseLiteLLM_ManagedVectorStore(TypedDict, total=False):
vector_store: LiteLLM_ManagedVectorStoresTable
vector_store: LiteLLM_ManagedVectorStoresTable

189
litellm/proxy/auth/auth_checks.py
View file

@ -1372,6 +1372,195 @@ async def get_team_object(
)
@log_db_metrics
async def get_team_object_by_alias(
team_alias: str,
prisma_client: Optional[PrismaClient],
user_api_key_cache: DualCache,
parent_otel_span: Optional["Span"] = None,
proxy_logging_obj: Optional[ProxyLogging] = None,
) -> LiteLLM_TeamTableCachedObj:
"""
Look up a team by its team_alias (name) in the database.
Args:
team_alias: The team name/alias to look up
prisma_client: Database client
user_api_key_cache: Cache for storing results
parent_otel_span: Optional OpenTelemetry span
proxy_logging_obj: Optional proxy logging object
Returns:
LiteLLM_TeamTableCachedObj: The team object if found
Raises:
HTTPException: If team doesn't exist or multiple teams have the same alias
"""
if prisma_client is None:
raise Exception(
"No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys"
)
# Check cache first (keyed by alias)
cache_key = "team_alias:{}".format(team_alias)
cached_team_obj = await _get_team_object_from_cache(
key=cache_key,
proxy_logging_obj=proxy_logging_obj,
user_api_key_cache=user_api_key_cache,
parent_otel_span=parent_otel_span,
)
if cached_team_obj is not None:
return cached_team_obj
# Query database by team_alias
try:
teams = await prisma_client.db.litellm_teamtable.find_many(
where={"team_alias": team_alias}
)
if not teams:
raise HTTPException(
status_code=404,
detail={
"error": f"Team with alias '{team_alias}' doesn't exist in db. Create team via `/team/new` call."
},
)
if len(teams) > 1:
raise HTTPException(
status_code=400,
detail={
"error": f"Multiple teams found with alias '{team_alias}'. Please use team_id_jwt_field instead or ensure team aliases are unique."
},
)
team = teams[0]
team_obj = LiteLLM_TeamTableCachedObj(**team.model_dump())
# Cache the result by both alias and team_id
await user_api_key_cache.async_set_cache(
key=cache_key,
value=team_obj,
ttl=DEFAULT_IN_MEMORY_TTL,
)
# Also cache by team_id for consistency
team_id_cache_key = "team_id:{}".format(team_obj.team_id)
await user_api_key_cache.async_set_cache(
key=team_id_cache_key,
value=team_obj,
ttl=DEFAULT_IN_MEMORY_TTL,
)
return team_obj
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception(
"Error looking up team by alias: %s", team_alias
)
raise HTTPException(
status_code=500,
detail={
"error": f"Error looking up team by alias '{team_alias}': {str(e)}"
},
)
@log_db_metrics
async def get_org_object_by_alias(
org_alias: str,
prisma_client: Optional[PrismaClient],
user_api_key_cache: DualCache,
parent_otel_span: Optional["Span"] = None,
proxy_logging_obj: Optional[ProxyLogging] = None,
) -> Optional[LiteLLM_OrganizationTable]:
"""
Look up an organization by its organization_alias in the database.
Args:
org_alias: The organization name/alias to look up
prisma_client: Database client
user_api_key_cache: Cache for storing results
parent_otel_span: Optional OpenTelemetry span
proxy_logging_obj: Optional proxy logging object
Returns:
LiteLLM_OrganizationTable if found, None otherwise
Raises:
HTTPException: If organization not found or multiple orgs have the same alias
"""
if prisma_client is None:
raise Exception(
"No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys"
)
# Check cache first (keyed by alias)
cache_key = "org_alias:{}".format(org_alias)
cached_org_obj = await user_api_key_cache.async_get_cache(key=cache_key)
if cached_org_obj is not None:
if isinstance(cached_org_obj, dict):
return LiteLLM_OrganizationTable(**cached_org_obj)
elif isinstance(cached_org_obj, LiteLLM_OrganizationTable):
return cached_org_obj
# Query database by organization_alias
try:
orgs = await prisma_client.db.litellm_organizationtable.find_many(
where={"organization_alias": org_alias}
)
if not orgs:
raise HTTPException(
status_code=404,
detail={
"error": f"Organization with alias '{org_alias}' doesn't exist in db. Create organization via `/organization/new` call."
},
)
if len(orgs) > 1:
raise HTTPException(
status_code=400,
detail={
"error": f"Multiple organizations found with alias '{org_alias}'. Please use org_id_jwt_field instead or ensure organization aliases are unique."
},
)
org = orgs[0]
org_obj = LiteLLM_OrganizationTable(**org.model_dump())
# Cache the result
await user_api_key_cache.async_set_cache(
key=cache_key,
value=org_obj.model_dump(),
ttl=DEFAULT_IN_MEMORY_TTL,
)
# Also cache by org_id for consistency
await user_api_key_cache.async_set_cache(
key="org_id:{}".format(org_obj.organization_id),
value=org_obj.model_dump(),
ttl=DEFAULT_IN_MEMORY_TTL,
)
return org_obj
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception(
"Error looking up organization by alias: %s", org_alias
)
raise HTTPException(
status_code=500,
detail={
"error": f"Error looking up organization by alias '{org_alias}': {str(e)}"
},
)
class ExperimentalUIJWTToken:
@staticmethod
def get_experimental_ui_login_jwt_auth_token(user_info: LiteLLM_UserTable) -> str:

139
litellm/proxy/auth/handle_jwt.py
View file

@ -48,10 +48,12 @@ from .auth_checks import (
get_actual_routes,
get_end_user_object,
get_org_object,
get_org_object_by_alias,
get_role_based_models,
get_role_based_routes,
get_team_membership,
get_team_object,
get_team_object_by_alias,
get_user_object,
)
@ -194,10 +196,13 @@ class JWTHandler:
def is_required_team_id(self) -> bool:
"""
Returns:
- True: if 'team_id_jwt_field' is set
- False: if not
- True: if 'team_id_jwt_field' or 'team_alias_jwt_field' is set
- False: if neither is set
"""
if self.litellm_jwtauth.team_id_jwt_field is None:
if (
self.litellm_jwtauth.team_id_jwt_field is None
and self.litellm_jwtauth.team_alias_jwt_field is None
):
return False
return True
@ -240,6 +245,31 @@ class JWTHandler:
team_id = default_value
return team_id
def get_team_alias(self, token: dict, default_value: Optional[str]) -> Optional[str]:
"""
Extract team name/alias from JWT token using the configured team_alias_jwt_field.
Args:
token: The decoded JWT token dictionary
default_value: Default value to return if field not found
Returns:
The team alias from the token, or default_value if not found
"""
try:
if self.litellm_jwtauth.team_alias_jwt_field is not None:
team_alias = get_nested_value(
data=token,
key_path=self.litellm_jwtauth.team_alias_jwt_field,
default=default_value,
)
return team_alias
else:
team_alias = None
except KeyError:
team_alias = default_value
return team_alias
def is_upsert_user_id(self, valid_user_email: Optional[bool] = None) -> bool:
"""
Returns:
@ -383,6 +413,31 @@ class JWTHandler:
org_id = default_value
return org_id
def get_org_alias(self, token: dict, default_value: Optional[str]) -> Optional[str]:
"""
Extract organization name/alias from JWT token using the configured org_alias_jwt_field.
Args:
token: The decoded JWT token dictionary
default_value: Default value to return if field not found
Returns:
The organization alias from the token, or default_value if not found
"""
try:
if self.litellm_jwtauth.org_alias_jwt_field is not None:
org_alias = get_nested_value(
data=token,
key_path=self.litellm_jwtauth.org_alias_jwt_field,
default=default_value,
)
return org_alias
else:
org_alias = None
except KeyError:
org_alias = default_value
return org_alias
def get_scopes(self, token: dict) -> List[str]:
try:
if isinstance(token["scope"], str):
@ -813,18 +868,14 @@ class JWTAuthManager:
parent_otel_span: Optional[Span],
proxy_logging_obj: ProxyLogging,
) -> Tuple[Optional[str], Optional[LiteLLM_TeamTable]]:
"""Find and validate specific team ID"""
"""Find and validate specific team ID from team_id_jwt_field or team_alias_jwt_field"""
individual_team_id = jwt_handler.get_team_id(
token=jwt_valid_token, default_value=None
)
if not individual_team_id and jwt_handler.is_required_team_id() is True:
raise Exception(
f"No team id found in token. Checked team_id field '{jwt_handler.litellm_jwtauth.team_id_jwt_field}'"
)
## VALIDATE TEAM OBJECT ###
team_object: Optional[LiteLLM_TeamTable] = None
# First try to get team by team_id
if individual_team_id:
team_object = await get_team_object(
team_id=individual_team_id,
@ -834,6 +885,37 @@ class JWTAuthManager:
proxy_logging_obj=proxy_logging_obj,
team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert,
)
return individual_team_id, team_object
# If no team_id found, try to resolve via team_alias_jwt_field
team_alias = jwt_handler.get_team_alias(
token=jwt_valid_token, default_value=None
)
if team_alias:
verbose_proxy_logger.info(
f"JWT Auth: Resolving team by alias: '{team_alias}'"
)
team_object = await get_team_object_by_alias(
team_alias=team_alias,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
if team_object:
individual_team_id = team_object.team_id
verbose_proxy_logger.info(
f"JWT Auth: Resolved team_alias='{team_alias}' to team_id='{individual_team_id}'"
)
return individual_team_id, team_object
# Check if team is required but not found
if jwt_handler.is_required_team_id() is True:
team_id_field = jwt_handler.litellm_jwtauth.team_id_jwt_field
team_alias_field = jwt_handler.litellm_jwtauth.team_alias_jwt_field
raise Exception(
f"No team found in token. Checked team_id field '{team_id_field}' and team_alias field '{team_alias_field}'"
)
return individual_team_id, team_object
@ -942,13 +1024,16 @@ class JWTAuthManager:
parent_otel_span: Optional[Span],
proxy_logging_obj: ProxyLogging,
route: str,
org_alias: Optional[str] = None,
) -> Tuple[
Optional[LiteLLM_UserTable],
Optional[LiteLLM_OrganizationTable],
Optional[LiteLLM_EndUserTable],
Optional[LiteLLM_EndUserTable],
Optional[LiteLLM_TeamMembership],
]:
"""Get user, org, and end user objects"""
"""Get user, org, and end user objects. Also resolves org aliases to IDs if configured."""
# Get org object - first try by ID, then by alias
org_object: Optional[LiteLLM_OrganizationTable] = None
if org_id:
org_object = (
@ -962,6 +1047,21 @@ class JWTAuthManager:
if org_id
else None
)
elif org_alias:
verbose_proxy_logger.info(
f"JWT Auth: Resolving org by alias: '{org_alias}'"
)
org_object = await get_org_object_by_alias(
org_alias=org_alias,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
if org_object:
verbose_proxy_logger.info(
f"JWT Auth: Resolved org_alias='{org_alias}' to org_id='{org_object.organization_id}'"
)
user_object: Optional[LiteLLM_UserTable] = None
if user_id:
@ -1304,6 +1404,8 @@ class JWTAuthManager:
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
# Extract alias fields for resolution (if configured)
org_alias = jwt_handler.get_org_alias(token=jwt_valid_token, default_value=None)
# Get other objects
user_object, org_object, end_user_object, team_membership_object = (
@ -1320,9 +1422,13 @@ class JWTAuthManager:
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
route=route,
org_alias=org_alias,
)
)
# Derive org_id from org_object if resolved by alias
resolved_org_id = org_object.organization_id if org_object else org_id
await JWTAuthManager.sync_user_role_and_teams(
jwt_handler=jwt_handler,
jwt_valid_token=jwt_valid_token,
@ -1345,10 +1451,9 @@ class JWTAuthManager:
)
# check if user is proxy admin
if user_object and user_object.user_role == LitellmUserRoles.PROXY_ADMIN:
is_proxy_admin = True
else:
is_proxy_admin = False
is_proxy_admin = bool(
user_object and user_object.user_role == LitellmUserRoles.PROXY_ADMIN
)
return JWTAuthBuilderResult(
is_proxy_admin=is_proxy_admin,
@ -1356,7 +1461,7 @@ class JWTAuthManager:
team_object=team_object,
user_id=user_id,
user_object=user_object,
org_id=org_id,
org_id=resolved_org_id, # Use resolved org_id (from alias lookup if applicable)
org_object=org_object,
end_user_id=end_user_id,
end_user_object=end_user_object,

64
litellm/proxy/auth/login_utils.py
View file

@ -34,6 +34,59 @@ from litellm.secret_managers.main import get_secret_bool
from litellm.types.proxy.ui_sso import ReturnedUITokenObject
async def expire_previous_ui_session_tokens(
user_id: str, prisma_client: Optional[PrismaClient]
) -> None:
"""
Expire (block) all other valid UI session tokens for a user.
This prevents accumulation of multiple valid UI session tokens that
are supposed to be short-lived test keys. Only affects keys with
team_id = "litellm-dashboard" and that haven't expired yet.
Args:
user_id: The user ID whose previous UI session tokens should be expired
prisma_client: Database client for performing the update
"""
if prisma_client is None:
return
try:
from datetime import datetime, timezone
current_time = datetime.now(timezone.utc)
# Find all unblocked AND non-expired UI session tokens for this user
ui_session_tokens = await prisma_client.db.litellm_verificationtoken.find_many(
where={
"user_id": user_id,
"team_id": "litellm-dashboard",
"OR": [
{"blocked": None}, # Tokens that have never been blocked (null)
{"blocked": False}, # Tokens explicitly set to not blocked
],
"expires": {"gt": current_time}, # Only get tokens that haven't expired
}
)
if not ui_session_tokens:
return
# Block all the found tokens
tokens_to_block = [token.token for token in ui_session_tokens if token.token]
if tokens_to_block:
await prisma_client.db.litellm_verificationtoken.update_many(
where={"token": {"in": tokens_to_block}},
data={"blocked": True}
)
except Exception:
# Silently fail - don't block login if cleanup fails
# This is a best-effort operation
pass
def get_ui_credentials(master_key: Optional[str]) -> tuple[str, str]:
"""
Get UI username and password from environment variables or master key.
@ -85,7 +138,7 @@ class LoginResult:
self.login_method = login_method
async def authenticate_user(
async def authenticate_user( # noqa: PLR0915
username: str,
password: str,
master_key: Optional[str],
@ -174,6 +227,10 @@ async def authenticate_user(
)
if os.getenv("DATABASE_URL") is not None:
# Expire any previous UI session tokens for this user
await expire_previous_ui_session_tokens(
user_id=key_user_id, prisma_client=prisma_client
)
response = await generate_key_helper_fn(
request_type="key",
**{
@ -260,6 +317,11 @@ async def authenticate_user(
hash_password, _password
):
if os.getenv("DATABASE_URL") is not None:
# Expire any previous UI session tokens for this user
await expire_previous_ui_session_tokens(
user_id=user_id, prisma_client=prisma_client
)
response = await generate_key_helper_fn(
request_type="key",
**{ # type: ignore

3
litellm/proxy/auth/user_api_key_auth.py
View file

@ -551,6 +551,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
valid_token = UserAPIKeyAuth(
api_key=None,
team_id=team_id,
team_alias=(
team_object.team_alias if team_object is not None else None
),
team_tpm_limit=(
team_object.tpm_limit if team_object is not None else None
),

31
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -449,6 +449,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
prepared_request.headers,
)
event_type = (
GuardrailEventHooks.pre_call
if source == "INPUT"
else GuardrailEventHooks.post_call
)
try:
httpx_response = await self.async_handler.post(
url=prepared_request.url,
@ -469,6 +475,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
# Re-raise the exception to maintain existing behavior
raise
@ -486,6 +493,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
#########################################################
if httpx_response.status_code == 200:
@ -605,10 +613,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
"""
Only raise exception for "BLOCKED" actions, not for "ANONYMIZED" actions.
If `self.mask_request_content` or `self.mask_response_content` is set to `True`,
If `self.mask_request_content` or `self.mask_response_content` is set to `True`,
then use the output from the guardrail to mask the request or response content.
However, even with masking enabled, content with action="BLOCKED" should still
However, even with masking enabled, content with action="BLOCKED" should still
raise an exception, only content with action="ANONYMIZED" should be masked.
"""
@ -731,9 +739,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
########## 1. Make the Bedrock API request ##########
#########################################################
bedrock_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = (
None
)
bedrock_guardrail_response: Optional[
Union[BedrockGuardrailResponse, str]
] = None
try:
bedrock_guardrail_response = await self.make_bedrock_api_request(
source="INPUT", messages=filtered_messages, request_data=data
@ -803,9 +811,9 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
########## 1. Make the Bedrock API request ##########
#########################################################
bedrock_guardrail_response: Optional[Union[BedrockGuardrailResponse, str]] = (
None
)
bedrock_guardrail_response: Optional[
Union[BedrockGuardrailResponse, str]
] = None
try:
bedrock_guardrail_response = await self.make_bedrock_api_request(
source="INPUT", messages=filtered_messages, request_data=data
@ -1296,11 +1304,6 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
request_data=request_data,
)
if bedrock_response.get("action") == "BLOCKED":
raise Exception(
f"Content blocked by Bedrock guardrail: {bedrock_response.get('reason', 'Unknown reason')}"
)
# Apply any masking that was applied by the guardrail
output_list = bedrock_response.get("output")

6
litellm/proxy/guardrails/guardrail_hooks/dynamoai/dynamoai.py
View file

@ -97,6 +97,7 @@ class DynamoAIGuardrails(CustomGuardrail):
async def _call_dynamoai_guardrails(
self,
messages: List[Dict[str, Any]],
event_type: GuardrailEventHooks,
text_type: str = "input",
request_data: Optional[dict] = None,
) -> DynamoAIResponse:
@ -157,6 +158,7 @@ class DynamoAIGuardrails(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
return response_json
@ -177,6 +179,7 @@ class DynamoAIGuardrails(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
raise
@ -332,6 +335,7 @@ class DynamoAIGuardrails(CustomGuardrail):
messages=_messages,
text_type="input",
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
verbose_proxy_logger.debug(
@ -380,6 +384,7 @@ class DynamoAIGuardrails(CustomGuardrail):
messages=_messages,
text_type="input",
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
verbose_proxy_logger.debug(
@ -460,6 +465,7 @@ class DynamoAIGuardrails(CustomGuardrail):
messages=dynamoai_messages,
text_type="output",
request_data=data,
event_type=GuardrailEventHooks.post_call,
)
verbose_proxy_logger.debug(

13
litellm/proxy/guardrails/guardrail_hooks/ibm_guardrails/ibm_detector.py
View file

@ -108,6 +108,7 @@ class IBMGuardrailDetector(CustomGuardrail):
async def _call_detector_server(
self,
contents: List[str],
event_type: GuardrailEventHooks,
request_data: Optional[dict] = None,
) -> List[List[IBMDetectorDetection]]:
"""
@ -142,7 +143,6 @@ class IBMGuardrailDetector(CustomGuardrail):
)
try:
response = await self.async_handler.post(
url=self.api_url,
json=payload,
@ -172,6 +172,7 @@ class IBMGuardrailDetector(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
return response_json
@ -192,6 +193,7 @@ class IBMGuardrailDetector(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
raise
@ -199,6 +201,7 @@ class IBMGuardrailDetector(CustomGuardrail):
async def _call_orchestrator(
self,
content: str,
event_type: GuardrailEventHooks,
request_data: Optional[dict] = None,
) -> List[IBMDetectorDetection]:
"""
@ -258,6 +261,7 @@ class IBMGuardrailDetector(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
return response_json.get("detections", [])
@ -278,6 +282,7 @@ class IBMGuardrailDetector(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
raise
@ -472,6 +477,7 @@ class IBMGuardrailDetector(CustomGuardrail):
result = await self._call_detector_server(
contents=contents_to_check,
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
verbose_proxy_logger.debug(
@ -500,6 +506,7 @@ class IBMGuardrailDetector(CustomGuardrail):
orchestrator_result = await self._call_orchestrator(
content=content,
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
verbose_proxy_logger.debug(
@ -557,6 +564,7 @@ class IBMGuardrailDetector(CustomGuardrail):
result = await self._call_detector_server(
contents=contents_to_check,
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
verbose_proxy_logger.debug(
@ -585,6 +593,7 @@ class IBMGuardrailDetector(CustomGuardrail):
orchestrator_result = await self._call_orchestrator(
content=content,
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
verbose_proxy_logger.debug(
@ -673,6 +682,7 @@ class IBMGuardrailDetector(CustomGuardrail):
result = await self._call_detector_server(
contents=contents_to_check,
request_data=data,
event_type=GuardrailEventHooks.post_call,
)
verbose_proxy_logger.debug(
@ -702,6 +712,7 @@ class IBMGuardrailDetector(CustomGuardrail):
orchestrator_result = await self._call_orchestrator(
content=content,
request_data=data,
event_type=GuardrailEventHooks.post_call,
)
verbose_proxy_logger.debug(

6
litellm/proxy/guardrails/guardrail_hooks/javelin/javelin.py
View file

@ -83,6 +83,7 @@ class JavelinGuardrail(CustomGuardrail):
async def call_javelin_guard(
self,
request: JavelinGuardRequest,
event_type: GuardrailEventHooks,
) -> JavelinGuardResponse:
"""
Call the Javelin guard API.
@ -158,6 +159,7 @@ class JavelinGuardrail(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
event_type=event_type,
)
async def async_pre_call_hook(
@ -208,7 +210,9 @@ class JavelinGuardrail(CustomGuardrail):
config=self.config if self.config else {},
)
javelin_response = await self.call_javelin_guard(request=javelin_guard_request)
javelin_response = await self.call_javelin_guard(
request=javelin_guard_request, event_type=GuardrailEventHooks.pre_call
)
assessments = javelin_response.get("assessments", [])
reject_prompt = ""

4
litellm/proxy/guardrails/guardrail_hooks/lakera_ai_v2.py
View file

@ -70,6 +70,7 @@ class LakeraAIGuardrail(CustomGuardrail):
self,
messages: List[AllMessageValues],
request_data: Dict,
event_type: GuardrailEventHooks,
) -> Tuple[LakeraAIResponse, Dict]:
"""
Call the Lakera AI v2 guard API.
@ -128,6 +129,7 @@ class LakeraAIGuardrail(CustomGuardrail):
end_time=datetime.now().timestamp(),
duration=(datetime.now() - start_time).total_seconds(),
masked_entity_count=masked_entity_count,
event_type=event_type,
)
def _mask_pii_in_messages(
@ -214,6 +216,7 @@ class LakeraAIGuardrail(CustomGuardrail):
lakera_guardrail_response, masked_entity_count = await self.call_v2_guard(
messages=new_messages,
request_data=data,
event_type=GuardrailEventHooks.pre_call,
)
#########################################################
@ -279,6 +282,7 @@ class LakeraAIGuardrail(CustomGuardrail):
lakera_guardrail_response, masked_entity_count = await self.call_v2_guard(
messages=new_messages,
request_data=data,
event_type=GuardrailEventHooks.during_call,
)
#########################################################

6
litellm/proxy/guardrails/guardrail_hooks/model_armor/model_armor.py
View file

@ -295,7 +295,9 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
filters = (
list(filter_results.values())
if isinstance(filter_results, dict)
else filter_results if isinstance(filter_results, list) else []
else filter_results
if isinstance(filter_results, list)
else []
)
# Prefer sanitized text from deidentifyResult if present
@ -327,6 +329,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
start_time: Optional[float] = None,
end_time: Optional[float] = None,
duration: Optional[float] = None,
event_type: Optional[GuardrailEventHooks] = None,
):
"""
Override to store only the Model Armor API response, not the entire data dict.
@ -351,6 +354,7 @@ class ModelArmorGuardrail(CustomGuardrail, VertexBase):
duration=duration,
start_time=start_time,
end_time=end_time,
event_type=event_type,
)
return response

37
litellm/proxy/guardrails/guardrail_hooks/noma/noma.py
View file

@ -119,9 +119,7 @@ class NomaGuardrail(CustomGuardrail):
self.api_base = api_base or os.environ.get(
"NOMA_API_BASE", NomaGuardrail._DEFAULT_API_BASE
)
self.application_id = application_id or os.environ.get(
"NOMA_APPLICATION_ID"
)
self.application_id = application_id or os.environ.get("NOMA_APPLICATION_ID")
self.default_application_id = "litellm"
if monitor_mode is None:
@ -163,6 +161,7 @@ class NomaGuardrail(CustomGuardrail):
self,
request_data: dict,
user_auth: UserAPIKeyAuth,
event_type: Optional[GuardrailEventHooks] = None,
) -> Optional[str]:
"""Shared logic for processing user message checks"""
start_time = datetime.now()
@ -213,6 +212,7 @@ class NomaGuardrail(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
if self.monitor_mode:
@ -242,6 +242,7 @@ class NomaGuardrail(CustomGuardrail):
request_data: dict,
response: LLMResponse,
user_auth: UserAPIKeyAuth,
event_type: Optional[GuardrailEventHooks] = None,
) -> Optional[str]:
"""Shared logic for processing LLM response checks"""
@ -293,6 +294,7 @@ class NomaGuardrail(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
if self.monitor_mode:
@ -578,7 +580,6 @@ class NomaGuardrail(CustomGuardrail):
data: dict,
call_type: CallTypesLiteral,
) -> Optional[Union[Exception, str, dict]]:
verbose_proxy_logger.debug("Running Noma pre-call hook")
if (
@ -602,7 +603,9 @@ class NomaGuardrail(CustomGuardrail):
return data
try:
return await self._check_user_message(data, user_api_key_dict)
return await self._check_user_message(
data, user_api_key_dict, GuardrailEventHooks.pre_call
)
except NomaBlockedMessage:
# Blocked requests were already logged in _process_user_message_check with "blocked" status
raise
@ -619,6 +622,7 @@ class NomaGuardrail(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=start_time.timestamp(),
duration=0.0,
event_type=GuardrailEventHooks.pre_call,
)
verbose_proxy_logger.error(f"Noma pre-call hook failed: {str(e)}")
@ -650,7 +654,9 @@ class NomaGuardrail(CustomGuardrail):
return data
try:
return await self._check_user_message(data, user_api_key_dict)
return await self._check_user_message(
data, user_api_key_dict, GuardrailEventHooks.during_call
)
except NomaBlockedMessage:
# Blocked requests were already logged in _process_user_message_check with "blocked" status
raise
@ -667,6 +673,7 @@ class NomaGuardrail(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=start_time.timestamp(),
duration=0.0,
event_type=GuardrailEventHooks.during_call,
)
verbose_proxy_logger.error(f"Noma moderation hook failed: {str(e)}")
@ -700,7 +707,9 @@ class NomaGuardrail(CustomGuardrail):
return response
try:
return await self._check_llm_response(data, response, user_api_key_dict)
return await self._check_llm_response(
data, response, user_api_key_dict, GuardrailEventHooks.post_call
)
except NomaBlockedMessage:
# Blocked requests were already logged in _process_llm_response_check with "blocked" status
raise
@ -717,6 +726,7 @@ class NomaGuardrail(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=start_time.timestamp(),
duration=0.0,
event_type=GuardrailEventHooks.post_call,
)
verbose_proxy_logger.error(f"Noma post-call hook failed: {str(e)}")
@ -728,9 +738,12 @@ class NomaGuardrail(CustomGuardrail):
self,
request_data: dict,
user_auth: UserAPIKeyAuth,
event_type: Optional[GuardrailEventHooks] = None,
) -> Union[Exception, str, dict, None]:
"""Check user message for policy violations"""
user_message = await self._process_user_message_check(request_data, user_auth)
user_message = await self._process_user_message_check(
request_data, user_auth, event_type
)
if not user_message:
return request_data
@ -741,10 +754,11 @@ class NomaGuardrail(CustomGuardrail):
request_data: dict,
response: LLMResponse,
user_auth: UserAPIKeyAuth,
event_type: Optional[GuardrailEventHooks] = None,
) -> Any:
"""Check LLM response for policy violations"""
content = await self._process_llm_response_check(
request_data, response, user_auth
request_data, response, user_auth, event_type
)
if not content:
return response
@ -858,7 +872,10 @@ class NomaGuardrail(CustomGuardrail):
if isinstance(assembled_model_response, ModelResponse):
try:
processed_response = await self._check_llm_response(
request_data, assembled_model_response, user_api_key_dict
request_data,
assembled_model_response,
user_api_key_dict,
GuardrailEventHooks.post_call,
)
except NomaBlockedMessage:
raise

24
litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py
View file

@ -24,6 +24,7 @@ from litellm.llms.custom_httpx.http_handler import (
httpxSpecialProvider,
)
from litellm.proxy._types import UserAPIKeyAuth
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import CallTypesLiteral, ModelResponse
if TYPE_CHECKING:
@ -523,6 +524,7 @@ class PanwPrismaAirsHandler(CustomGuardrail):
scan_result: Dict[str, Any],
data: Dict[str, Any],
start_time: datetime,
event_type: GuardrailEventHooks,
is_response: bool = False,
) -> Optional[Dict[str, Any]]:
"""Handle API errors with fail-open/fail-closed logic."""
@ -542,6 +544,7 @@ class PanwPrismaAirsHandler(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=duration,
event_type=event_type,
)
if scan_result.get("_always_block"):
@ -735,7 +738,11 @@ class PanwPrismaAirsHandler(CustomGuardrail):
if scan_result.get("_is_transient") or scan_result.get("_always_block"):
return self._handle_api_error_with_logging(
scan_result, data, start_time, is_response=False
scan_result,
data,
start_time,
is_response=False,
event_type=GuardrailEventHooks.pre_call,
)
end_time = datetime.now()
@ -749,6 +756,7 @@ class PanwPrismaAirsHandler(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=(end_time - start_time).total_seconds(),
event_type=GuardrailEventHooks.pre_call,
)
action = scan_result.get("action", "block")
@ -872,7 +880,11 @@ class PanwPrismaAirsHandler(CustomGuardrail):
if scan_result.get("_is_transient") or scan_result.get("_always_block"):
self._handle_api_error_with_logging(
scan_result, data, start_time, is_response=True
scan_result,
data,
start_time,
is_response=True,
event_type=GuardrailEventHooks.post_call,
)
return response
@ -887,6 +899,7 @@ class PanwPrismaAirsHandler(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=(end_time - start_time).total_seconds(),
event_type=GuardrailEventHooks.post_call,
)
action = scan_result.get("action", "block")
@ -1066,7 +1079,11 @@ class PanwPrismaAirsHandler(CustomGuardrail):
if scan_result.get("_is_transient") or scan_result.get("_always_block"):
self._handle_api_error_with_logging(
scan_result, request_data, start_time, is_response=True
scan_result,
request_data,
start_time,
is_response=True,
event_type=EventHooks.post_call,
)
for chunk in all_chunks:
yield chunk
@ -1083,6 +1100,7 @@ class PanwPrismaAirsHandler(CustomGuardrail):
start_time=start_time.timestamp(),
end_time=end_time.timestamp(),
duration=(end_time - start_time).total_seconds(),
event_type=EventHooks.post_call,
)
# Add guardrail to applied guardrails header for observability

17
litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
View file

@ -28,7 +28,6 @@ class UnifiedLLMGuardrails(CustomLogger):
self,
**kwargs,
):
# store kwargs as optional_params
self.optional_params = kwargs
@ -63,6 +62,9 @@ class UnifiedLLMGuardrails(CustomLogger):
return data
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call
if call_type == CallTypes.call_mcp_tool.value:
event_type = GuardrailEventHooks.pre_mcp_call
if (
guardrail_to_apply.should_run_guardrail(data=data, event_type=event_type)
is not True
@ -114,6 +116,9 @@ class UnifiedLLMGuardrails(CustomLogger):
return data
event_type: GuardrailEventHooks = GuardrailEventHooks.during_call
if call_type == CallTypes.call_mcp_tool.value:
event_type = GuardrailEventHooks.during_mcp_call
if (
guardrail_to_apply.should_run_guardrail(data=data, event_type=event_type)
is not True
@ -128,7 +133,10 @@ class UnifiedLLMGuardrails(CustomLogger):
endpoint_guardrail_translation_mappings = (
load_guardrail_translation_mappings()
)
if call_type is not None and CallTypes(call_type) not in endpoint_guardrail_translation_mappings:
if (
call_type is not None
and CallTypes(call_type) not in endpoint_guardrail_translation_mappings
):
return data
endpoint_translation = endpoint_guardrail_translation_mappings[
@ -180,8 +188,8 @@ class UnifiedLLMGuardrails(CustomLogger):
call_type: Optional[CallTypesLiteral] = None
if user_api_key_dict.request_route is not None:
call_types = get_call_types_for_route(user_api_key_dict.request_route)
if call_types is not None and len(call_types) > 0: # type: ignore
call_type = call_types[0] # type: ignore
if call_types is not None and len(call_types) > 0: # type: ignore
call_type = call_types[0] # type: ignore
if call_type is None:
call_type = _infer_call_type(call_type=None, completion_response=response) # type: ignore
@ -330,7 +338,6 @@ class UnifiedLLMGuardrails(CustomLogger):
# Process chunk based on sampling rate
if chunk_counter % sampling_rate == 0:
verbose_proxy_logger.debug(
"Processing streaming chunk %s (sampling_rate=%s) with guardrail %s",
chunk_counter,

24
litellm/proxy/management_endpoints/budget_management_endpoints.py
View file

@ -55,6 +55,18 @@ async def new_budget(
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
# Validate budget values are not negative
if budget_obj.max_budget is not None and budget_obj.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {budget_obj.max_budget}"}
)
if budget_obj.soft_budget is not None and budget_obj.soft_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"soft_budget cannot be negative. Received: {budget_obj.soft_budget}"}
)
# if no budget_reset_at date is set, but a budget_duration is given, then set budget_reset_at initially to the first completed duration interval in future
if budget_obj.budget_reset_at is None and budget_obj.budget_duration is not None:
budget_obj.budget_reset_at = datetime.utcnow() + timedelta(
@ -107,6 +119,18 @@ async def update_budget(
if budget_obj.budget_id is None:
raise HTTPException(status_code=400, detail={"error": "budget_id is required"})
# Validate budget values are not negative
if budget_obj.max_budget is not None and budget_obj.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {budget_obj.max_budget}"}
)
if budget_obj.soft_budget is not None and budget_obj.soft_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"soft_budget cannot be negative. Received: {budget_obj.soft_budget}"}
)
response = await prisma_client.db.litellm_budgettable.update(
where={"budget_id": budget_obj.budget_id},
data={

137
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -1069,6 +1069,18 @@ async def generate_key_fn(
verbose_proxy_logger.debug("entered /key/generate")
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
if data.soft_budget is not None and data.soft_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"soft_budget cannot be negative. Received: {data.soft_budget}"}
)
if user_custom_key_generate is not None:
if asyncio.iscoroutinefunction(user_custom_key_generate):
result = await user_custom_key_generate(data) # type: ignore
@ -1502,6 +1514,13 @@ async def update_key_fn(
)
try:
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
data_json: dict = data.model_dump(exclude_unset=True, exclude_none=True)
key = data_json.pop("key")
@ -3020,10 +3039,14 @@ async def list_keys(
description="Column to sort by (e.g. 'user_id', 'created_at', 'spend')",
),
sort_order: str = Query(default="desc", description="Sort order ('asc' or 'desc')"),
expand: Optional[List[str]] = Query(None, description="Expand related objects (e.g. 'user')"),
) -> KeyListResponseObject:
"""
List all keys for a given user / team / organization.
Parameters:
expand: Optional[List[str]] - Expand related objects (e.g. 'user' to include user information)
Returns:
{
"keys": List[str] or List[UserAPIKeyAuth],
@ -3031,6 +3054,9 @@ async def list_keys(
"current_page": int,
"total_pages": int,
}
When expand includes "user", each key object will include a "user" field with the associated user object.
Note: When expand=user is specified, full key objects are returned regardless of the return_full_object parameter.
"""
try:
from litellm.proxy.proxy_server import prisma_client
@ -3080,6 +3106,7 @@ async def list_keys(
include_created_by_keys=include_created_by_keys,
sort_by=sort_by,
sort_order=sort_order,
expand=expand,
)
verbose_proxy_logger.debug("Successfully prepared response")
@ -3215,45 +3242,17 @@ def _validate_sort_params(
return order_by
async def _list_key_helper(
prisma_client: PrismaClient,
page: int,
size: int,
def _build_key_filter_conditions(
user_id: Optional[str],
team_id: Optional[str],
organization_id: Optional[str],
key_alias: Optional[str],
key_hash: Optional[str],
exclude_team_id: Optional[str] = None,
return_full_object: bool = False,
admin_team_ids: Optional[
List[str]
] = None, # New parameter for teams where user is admin
include_created_by_keys: bool = False,
sort_by: Optional[str] = None,
sort_order: str = "desc",
) -> KeyListResponseObject:
"""
Helper function to list keys
Args:
page: int
size: int
user_id: Optional[str]
team_id: Optional[str]
key_alias: Optional[str]
exclude_team_id: Optional[str] # exclude a specific team_id
return_full_object: bool # when true, will return UserAPIKeyAuth objects instead of just the token
admin_team_ids: Optional[List[str]] # list of team IDs where the user is an admin
Returns:
KeyListResponseObject
{
"keys": List[str] or List[UserAPIKeyAuth], # Updated to reflect possible return types
"total_count": int,
"current_page": int,
"total_pages": int,
}
"""
exclude_team_id: Optional[str],
admin_team_ids: Optional[List[str]],
include_created_by_keys: bool,
) -> Dict[str, Union[str, Dict[str, Any], List[Dict[str, Any]]]]:
"""Build filter conditions for key listing."""
# Prepare filter conditions
where: Dict[str, Union[str, Dict[str, Any], List[Dict[str, Any]]]] = {}
where.update(_get_condition_to_filter_out_ui_session_tokens())
@ -3294,6 +3293,59 @@ async def _list_key_helper(
where.update(or_conditions[0])
verbose_proxy_logger.debug(f"Filter conditions: {where}")
return where
async def _list_key_helper(
prisma_client: PrismaClient,
page: int,
size: int,
user_id: Optional[str],
team_id: Optional[str],
organization_id: Optional[str],
key_alias: Optional[str],
key_hash: Optional[str],
exclude_team_id: Optional[str] = None,
return_full_object: bool = False,
admin_team_ids: Optional[
List[str]
] = None, # New parameter for teams where user is admin
include_created_by_keys: bool = False,
sort_by: Optional[str] = None,
sort_order: str = "desc",
expand: Optional[List[str]] = None,
) -> KeyListResponseObject:
"""
Helper function to list keys
Args:
page: int
size: int
user_id: Optional[str]
team_id: Optional[str]
key_alias: Optional[str]
exclude_team_id: Optional[str] # exclude a specific team_id
return_full_object: bool # when true, will return UserAPIKeyAuth objects instead of just the token
admin_team_ids: Optional[List[str]] # list of team IDs where the user is an admin
Returns:
KeyListResponseObject
{
"keys": List[str] or List[UserAPIKeyAuth], # Updated to reflect possible return types
"total_count": int,
"current_page": int,
"total_pages": int,
}
"""
where = _build_key_filter_conditions(
user_id=user_id,
team_id=team_id,
organization_id=organization_id,
key_alias=key_alias,
key_hash=key_hash,
exclude_team_id=exclude_team_id,
admin_team_ids=admin_team_ids,
include_created_by_keys=include_created_by_keys,
)
# Calculate skip for pagination
skip = (page - 1) * size
@ -3334,13 +3386,28 @@ async def _list_key_helper(
# Calculate total pages
total_pages = -(-total_count // size) # Ceiling division
# Fetch user information if expand includes "user"
user_map = {}
if expand and "user" in expand:
user_ids = [key.user_id for key in keys if key.user_id]
if user_ids:
users = await prisma_client.db.litellm_usertable.find_many(
where={"user_id": {"in": list(set(user_ids))}} # Remove duplicates
)
user_map = {user.user_id: user for user in users}
# Prepare response
key_list: List[Union[str, UserAPIKeyAuth]] = []
for key in keys:
key_dict = key.dict()
# Attach object_permission if object_permission_id is set
key_dict = await attach_object_permission_to_dict(key_dict, prisma_client)
if return_full_object is True:
# Include user information if expand includes "user"
if expand and "user" in expand and key.user_id and key.user_id in user_map:
key_dict["user"] = user_map[key.user_id].dict()
if return_full_object is True or (expand and "user" in expand):
key_list.append(UserAPIKeyAuth(**key_dict)) # Return full key object
else:
_token = key_dict.get("token")

187
litellm/proxy/management_endpoints/mcp_management_endpoints.py
View file

@ -16,7 +16,7 @@ Endpoints here:
import importlib
from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import Any, Dict, Iterable, List, Optional
from typing import Any, Dict, Iterable, List, Literal, Optional
from fastapi import (
APIRouter,
@ -24,6 +24,7 @@ from fastapi import (
Form,
Header,
HTTPException,
Query,
Request,
Response,
status,
@ -208,6 +209,9 @@ if MCP_AVAILABLE:
command=payload.command,
args=payload.args,
env=payload.env,
authorization_url=payload.authorization_url,
token_url=payload.token_url,
registration_url=payload.registration_url,
)
def get_prisma_client_or_throw(message: str):
@ -296,117 +300,6 @@ if MCP_AVAILABLE:
access_groups_list = sorted(list(access_groups))
return {"access_groups": access_groups_list}
@router.get(
"/server/{server_id}/health",
description="Perform health check on a specific MCP server",
dependencies=[Depends(user_api_key_auth)],
)
async def health_check_mcp_server(
server_id: str,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Perform a health check on the MCP server specified by the `server_id`
Parameters:
- server_id: str - Required. The unique identifier of the mcp server to health check.
```
curl --location 'http://localhost:4000/v1/mcp/server/{server_id}/health' \
--header 'Authorization: Bearer your_api_key_here'
```
"""
# Check if server exists and user has access
prisma_client = get_prisma_client_or_throw(
"Database not connected. Connect a database to your proxy"
)
# check to see if server exists for all users
mcp_server = await get_mcp_server(prisma_client, server_id)
if mcp_server is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={"error": f"MCP Server with id {server_id} not found"},
)
# Implement authz restriction from requested user
if not _user_has_admin_view(user_api_key_dict):
# Perform authz check to filter the mcp servers user has access to
mcp_server_records = await get_all_mcp_servers_for_user(
prisma_client, user_api_key_dict
)
exists = does_mcp_server_exist(mcp_server_records, server_id)
if not exists:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail={
"error": f"User does not have permission to access mcp server with id {server_id}. You can only access mcp servers that you have access to."
},
)
# Perform health check using server manager
try:
health_result = await global_mcp_server_manager.health_check_server(
server_id
)
return health_result
except Exception as e:
verbose_proxy_logger.exception(
f"Error performing health check on MCP server {server_id}: {str(e)}"
)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={"error": f"Error performing health check: {str(e)}"},
)
@router.get(
"/server/health",
description="Perform health check on all accessible MCP servers",
dependencies=[Depends(user_api_key_auth)],
)
async def health_check_all_mcp_servers(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Perform health checks on all MCP servers accessible to the user
```
curl --location 'http://localhost:4000/v1/mcp/server/health' \
--header 'Authorization: Bearer your_api_key_here'
```
"""
# Use server manager to get health checks for allowed servers
try:
all_health_results = (
await global_mcp_server_manager.health_check_allowed_servers(
user_api_key_auth=user_api_key_dict
)
)
return {
"total_servers": len(all_health_results),
"healthy_count": len(
[r for r in all_health_results.values() if r["status"] == "healthy"]
),
"unhealthy_count": len(
[
r
for r in all_health_results.values()
if r["status"] == "unhealthy"
]
),
"unknown_count": len(
[r for r in all_health_results.values() if r["status"] == "unknown"]
),
"servers": all_health_results,
}
except Exception as e:
verbose_proxy_logger.exception(
f"Error performing health checks on MCP servers: {str(e)}"
)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={"error": f"Error performing health checks: {str(e)}"},
)
## FastAPI Routes
@router.get(
"/server",
@ -429,7 +322,7 @@ if MCP_AVAILABLE:
aggregated_servers: Dict[str, LiteLLM_MCPServerTable] = {}
for auth_context in auth_contexts:
servers = await global_mcp_server_manager.get_all_mcp_servers_with_health_and_teams(
servers = await global_mcp_server_manager.get_all_allowed_mcp_servers(
user_api_key_auth=auth_context
)
for server in servers:
@ -447,6 +340,56 @@ if MCP_AVAILABLE:
server.mcp_info["is_public"] = True
return redacted_mcp_servers
@router.get(
"/server/health",
description="Health check for MCP servers",
dependencies=[Depends(user_api_key_auth)],
)
async def health_check_servers(
server_ids: Optional[List[str]] = Query(
None,
description="Server IDs to check. If not provided, checks all accessible servers.",
),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Perform health checks on one or more MCP servers.
Parameters:
- server_ids: Optional list of server IDs. If not provided, checks all accessible servers.
Returns:
- Health check results for requested servers
```
# Check all accessible servers
curl --location 'http://localhost:4000/v1/mcp/server/health' \
--header 'Authorization: Bearer your_api_key_here'
# Check specific servers
curl --location 'http://localhost:4000/v1/mcp/server/health?server_ids=server-1&server_ids=server-2' \
--header 'Authorization: Bearer your_api_key_here'
```
"""
auth_contexts = await build_effective_auth_contexts(user_api_key_dict)
server_status_map: Dict[
str, Optional[Literal["healthy", "unhealthy", "unknown"]]
] = {}
for auth_context in auth_contexts:
servers = await global_mcp_server_manager.get_all_mcp_servers_with_health_and_teams(
user_api_key_auth=auth_context,
server_ids=server_ids,
)
for server in servers:
if server.server_id not in server_status_map:
server_status_map[server.server_id] = server.status
return [
{"server_id": server_id, "status": status}
for server_id, status in server_status_map.items()
]
@router.get(
"/server/{server_id}",
description="Returns the mcp server info",
@ -484,15 +427,11 @@ if MCP_AVAILABLE:
server_id
)
# Update the server object with health check results
mcp_server.status = health_result.get("status", "unknown")
mcp_server.last_health_check = (
datetime.fromisoformat(
health_result.get("last_health_check", datetime.now().isoformat())
)
if health_result.get("last_health_check")
else None
mcp_server.status = (
health_result.status if health_result.status else "unknown"
)
mcp_server.health_check_error = health_result.get("error")
mcp_server.last_health_check = health_result.last_health_check
mcp_server.health_check_error = health_result.health_check_error
except Exception as e:
verbose_proxy_logger.debug(
f"Error performing health check on server {server_id}: {e}"
@ -512,7 +451,7 @@ if MCP_AVAILABLE:
exists = does_mcp_server_exist(mcp_server_records, server_id)
if exists:
await global_mcp_server_manager.add_update_server(mcp_server)
await global_mcp_server_manager.add_server(mcp_server)
return _redact_mcp_credentials(mcp_server)
else:
raise HTTPException(
@ -586,7 +525,7 @@ if MCP_AVAILABLE:
payload,
touched_by=user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
)
await global_mcp_server_manager.add_update_server(new_mcp_server)
await global_mcp_server_manager.add_server(new_mcp_server)
# Ensure registry is up to date by reloading from database
await global_mcp_server_manager.reload_servers_from_database()
@ -867,7 +806,7 @@ if MCP_AVAILABLE:
"error": f"MCP Server not found, passed server_id={payload.server_id}"
},
)
await global_mcp_server_manager.add_update_server(mcp_server_record_updated)
await global_mcp_server_manager.update_server(mcp_server_record_updated)
# Ensure registry is up to date by reloading from database
await global_mcp_server_manager.reload_servers_from_database()

34
litellm/proxy/management_endpoints/organization_endpoints.py
View file

@ -24,8 +24,11 @@ from litellm.proxy.management_endpoints.budget_management_endpoints import (
new_budget,
update_budget,
)
from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity
from litellm.proxy.management_endpoints.common_utils import (
_set_object_metadata_field,
_user_has_admin_view,
)
from litellm.proxy.management_helpers.object_permission_utils import (
handle_update_object_permission_common,
)
@ -34,11 +37,10 @@ from litellm.proxy.management_helpers.utils import (
management_endpoint_wrapper,
)
from litellm.proxy.utils import PrismaClient
from litellm.utils import _update_dictionary
from litellm.types.proxy.management_endpoints.common_daily_activity import (
SpendAnalyticsPaginatedResponse,
)
from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity
from litellm.utils import _update_dictionary
router = APIRouter()
@ -168,6 +170,18 @@ async def new_organization(
status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}
)
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
if data.soft_budget is not None and data.soft_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"soft_budget cannot be negative. Received: {data.soft_budget}"}
)
user_object_correct_type: Optional[LiteLLM_UserTable] = None
if user_api_key_dict.user_id is not None:
@ -414,6 +428,18 @@ async def update_organization(
# Create validated data model
data = LiteLLM_OrganizationTableUpdate(**raw_data_with_flat_budget_fields)
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
if data.soft_budget is not None and data.soft_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"soft_budget cannot be negative. Received: {data.soft_budget}"}
)
if data.updated_by is None:
data.updated_by = user_api_key_dict.user_id

26
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -732,6 +732,18 @@ async def new_team( # noqa: PLR0915
if prisma_client is None:
raise HTTPException(status_code=500, detail={"error": "No db connected"})
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
if data.team_member_budget is not None and data.team_member_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"team_member_budget cannot be negative. Received: {data.team_member_budget}"}
)
# Check if license is over limit
total_teams = await prisma_client.db.litellm_teamtable.count()
if total_teams and _license_check.is_team_count_over_limit(
@ -1169,7 +1181,7 @@ def validate_team_org_change(
"/team/update", tags=["team management"], dependencies=[Depends(user_api_key_auth)]
)
@management_endpoint_wrapper
async def update_team(
async def update_team( # noqa: PLR0915
data: UpdateTeamRequest,
http_request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@ -1254,6 +1266,18 @@ async def update_team(
raise HTTPException(status_code=400, detail={"error": "No team id passed in"})
verbose_proxy_logger.debug("/team/update - %s", data)
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
if data.team_member_budget is not None and data.team_member_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"team_member_budget cannot be negative. Received: {data.team_member_budget}"}
)
existing_team_row = await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": data.team_id}
)

25
litellm/proxy/proxy_server.py
View file

@ -4626,7 +4626,7 @@ class ProxyStartupEvent:
verbose_proxy_logger.info("Batch cost check job scheduled successfully")
except Exception as e:
verbose_proxy_logger.error(f"Failed to setup batch cost checking: {e}")
verbose_proxy_logger.debug(f"Failed to setup batch cost checking: {e}")
verbose_proxy_logger.debug(
"Checking batch cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..."
)
@ -4657,7 +4657,7 @@ class ProxyStartupEvent:
verbose_proxy_logger.info("Responses cost check job scheduled successfully")
except Exception as e:
verbose_proxy_logger.error(f"Failed to setup responses cost checking: {e}")
verbose_proxy_logger.debug(f"Failed to setup responses cost checking: {e}")
verbose_proxy_logger.debug(
"Checking responses cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..."
)
@ -7322,13 +7322,9 @@ async def model_info_v2(
"""
global llm_model_list, general_settings, user_config_file_path, proxy_config, llm_router
if llm_router is None:
raise HTTPException(
status_code=500,
detail={
"error": f"No model list passed, models router={llm_router}. You can add a model through the config.yaml or on the LiteLLM Admin UI."
},
)
# Return empty data array when no models are configured (graceful handling for fresh installs)
if llm_router is None or not llm_router.model_list:
return {"data": []}
if prisma_client is None:
raise HTTPException(
@ -8228,14 +8224,9 @@ async def model_group_info(
"""
global llm_model_list, general_settings, user_config_file_path, proxy_config, llm_router
if llm_model_list is None:
raise HTTPException(
status_code=500, detail={"error": "LLM Model List not loaded in"}
)
if llm_router is None:
raise HTTPException(
status_code=500, detail={"error": "LLM Router is not loaded in"}
)
# Return empty data array when no models are configured (graceful handling for fresh installs)
if llm_model_list is None or llm_router is None or not llm_model_list:
return {"data": []}
from litellm.proxy.utils import get_available_models_for_user

74
litellm/proxy/public_endpoints/provider_create_fields.json
View file

@ -1680,6 +1680,34 @@
],
"default_model_placeholder": "gpt-3.5-turbo"
},
{
"provider": "MINIMAX",
"provider_display_name": "MiniMax",
"litellm_provider": "minimax",
"credential_fields": [
{
"key": "api_key",
"label": "API Key",
"placeholder": "your-minimax-api-key",
"tooltip": "MiniMax API Key from https://platform.minimaxi.com/",
"required": true,
"field_type": "password",
"options": null,
"default_value": null
},
{
"key": "api_base",
"label": "API Base URL",
"placeholder": "https://api.minimax.io/v1",
"tooltip": "International: https://api.minimax.io/v1, China: https://api.minimaxi.com/v1",
"required": false,
"field_type": "text",
"options": null,
"default_value": "https://api.minimax.io/v1"
}
],
"default_model_placeholder": "minimax/MiniMax-M2"
},
{
"provider": "MOONSHOT",
"provider_display_name": "Moonshot",
@ -2865,7 +2893,7 @@
"key": "api_base",
"label": "API Base",
"placeholder": null,
"tooltip": null,
"tooltip": "Base URL of your WatsonX instance",
"required": false,
"field_type": "text",
"options": null,
@ -2875,14 +2903,54 @@
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"tooltip": "IBM Cloud API key. Required if not using Token or Zen API Key",
"required": false,
"field_type": "password",
"options": null,
"default_value": null
},
{
"key": "token",
"label": "IAM Token",
"placeholder": null,
"tooltip": "Pre-generated IAM bearer token. Use instead of API Key if you manage tokens externally",
"required": false,
"field_type": "password",
"options": null,
"default_value": null
},
{
"key": "zen_api_key",
"label": "Zen API Key",
"placeholder": null,
"tooltip": "Zen API Key for Cloud Pak for Data deployments. Use instead of API Key for on-premises",
"required": false,
"field_type": "password",
"options": null,
"default_value": null
},
{
"key": "project_id",
"label": "Project ID",
"placeholder": null,
"tooltip": "Optional: Your Watsonx.ai Project ID",
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "space_id",
"label": "Deployment Space ID",
"placeholder": null,
"tooltip": "Optional: Watsonx.ai Deployment Space ID",
"required": false,
"field_type": "text",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "gpt-3.5-turbo"
"default_model_placeholder": "watsonx/ibm/granite-3-3-8b-instruct"
},
{
"provider": "WATSONX_TEXT",

3
litellm/proxy/schema.prisma
View file

@ -208,6 +208,9 @@ model LiteLLM_MCPServerTable {
command String?
args String[] @default([])
env Json? @default("{}")
authorization_url String?
token_url String?
registration_url String?
}
// Generate Tokens for Proxy

18
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -11,7 +11,10 @@ from pydantic import BaseModel
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB, REDACTED_BY_LITELM_STRING
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.litellm_core_utils.core_helpers import (
get_litellm_metadata_from_kwargs,
reconstruct_model_name,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
from litellm.proxy.utils import PrismaClient, hash_token
@ -100,9 +103,9 @@ def _get_spend_logs_metadata(
clean_metadata["applied_guardrails"] = applied_guardrails
clean_metadata["batch_models"] = batch_models
clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata
clean_metadata["vector_store_request_metadata"] = (
_get_vector_store_request_for_spend_logs_payload(vector_store_request_metadata)
)
clean_metadata[
"vector_store_request_metadata"
] = _get_vector_store_request_for_spend_logs_payload(vector_store_request_metadata)
clean_metadata["guardrail_information"] = guardrail_information
clean_metadata["usage_object"] = usage_object
clean_metadata["model_map_information"] = model_map_information
@ -393,6 +396,9 @@ def get_logging_payload( # noqa: PLR0915
# Extract agent_id for A2A requests (set directly on model_call_details)
agent_id: Optional[str] = kwargs.get("agent_id")
custom_llm_provider = kwargs.get("custom_llm_provider")
raw_model = cast(str, kwargs.get("model") or "")
model_name = reconstruct_model_name(raw_model, custom_llm_provider, metadata or {})
try:
payload: SpendLogsPayload = SpendLogsPayload(
@ -403,7 +409,7 @@ def get_logging_payload( # noqa: PLR0915
startTime=_ensure_datetime_utc(start_time),
endTime=_ensure_datetime_utc(end_time),
completionStartTime=_ensure_datetime_utc(completion_start_time),
model=kwargs.get("model", "") or "",
model=model_name,
user=metadata.get("user_api_key_user_id", "") or "",
team_id=metadata.get("user_api_key_team_id", "") or "",
organization_id=metadata.get("user_api_key_org_id") or "",
@ -449,7 +455,7 @@ def get_logging_payload( # noqa: PLR0915
# Explicitly clear large intermediate objects to reduce memory pressure
del response_obj_dict, usage, clean_metadata, additional_usage_values
return payload
except Exception as e:
verbose_proxy_logger.exception(

72
litellm/proxy/utils.py
View file

@ -151,25 +151,25 @@ def _get_email_logger_class():
"""
Determine which email logger class to use based on environment variables.
Priority: SendGrid > Resend > SMTP > BaseEmailLogger (fallback)
Returns:
The email logger class to use, or None if BaseEmailLogger is not available
"""
if BaseEmailLogger is None:
return None
# Check for SendGrid API key
if SendGridEmailLogger is not None and os.getenv("SENDGRID_API_KEY"):
return SendGridEmailLogger
# Check for Resend API key
if ResendEmailLogger is not None and os.getenv("RESEND_API_KEY"):
return ResendEmailLogger
# Check for SMTP configuration
if SMTPEmailLogger is not None and os.getenv("SMTP_HOST"):
return SMTPEmailLogger
# Fallback to BaseEmailLogger (though it won't actually send emails)
return BaseEmailLogger
@ -452,7 +452,6 @@ class ProxyLogging:
litellm.logging_callback_manager.add_litellm_callback(self.service_logging_obj) # type: ignore
for callback in litellm.callbacks:
if isinstance(callback, str):
callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class( # type: ignore
cast(_custom_logger_compatible_callbacks_literal, callback),
internal_usage_cache=self.internal_usage_cache.dual_cache,
@ -965,7 +964,7 @@ class ProxyLogging:
# Determine the event type based on call type
event_type = GuardrailEventHooks.pre_call
if call_type == "mcp_call":
if call_type == CallTypes.call_mcp_tool.value:
event_type = GuardrailEventHooks.pre_mcp_call
# Check if the guardrail should run for this request
@ -1038,7 +1037,6 @@ class ProxyLogging:
data.pop("prompt_id", None)
if custom_logger and prompt_spec is not None:
(
model,
messages,
@ -1261,7 +1259,7 @@ class ProxyLogging:
from litellm.types.guardrails import GuardrailEventHooks
event_type = GuardrailEventHooks.during_call
if call_type == "mcp_call":
if call_type == CallTypes.call_mcp_tool.value:
event_type = GuardrailEventHooks.during_mcp_call
if (
@ -1270,7 +1268,7 @@ class ProxyLogging:
):
continue
# Convert user_api_key_dict to proper format for async_moderation_hook
if call_type == "mcp_call":
if call_type == CallTypes.call_mcp_tool.value:
user_api_key_auth_dict = self._convert_user_api_key_auth_to_dict(
user_api_key_dict
)
@ -1288,7 +1286,6 @@ class ProxyLogging:
call_type=call_type,
)
else:
guardrail_task = callback.async_moderation_hook(
data=data,
user_api_key_dict=user_api_key_auth_dict, # type: ignore
@ -1337,7 +1334,7 @@ class ProxyLogging:
if self.alerting is None:
# do nothing if alerting is not switched on
return
if "slack" in self.alerting:
await self.slack_alerting_instance.budget_alerts(
type=type,
@ -1548,7 +1545,10 @@ class ProxyLogging:
traceback_str=traceback_str,
)
# If callback returned an HTTPException, use it (first one wins)
if isinstance(hook_result, HTTPException) and transformed_exception is None:
if (
isinstance(hook_result, HTTPException)
and transformed_exception is None
):
transformed_exception = hook_result
except HTTPException as e:
# If callback raised an HTTPException, use it (first one wins)
@ -1849,7 +1849,6 @@ class ProxyLogging:
current_response = response
for callback in litellm.callbacks:
_callback: Optional[CustomLogger] = None
if isinstance(callback, str):
_callback = litellm.litellm_core_utils.litellm_logging.get_custom_logger_compatible_class(
@ -3568,11 +3567,13 @@ class ProxyUpdateSpend:
)
# Atomically read and remove logs to process (protected by lock)
async with prisma_client._spend_log_transactions_lock:
logs_to_process = prisma_client.spend_log_transactions[:MAX_LOGS_PER_INTERVAL]
logs_to_process = prisma_client.spend_log_transactions[
:MAX_LOGS_PER_INTERVAL
]
# Remove the logs we're about to process
prisma_client.spend_log_transactions = (
prisma_client.spend_log_transactions[len(logs_to_process):]
)
prisma_client.spend_log_transactions = prisma_client.spend_log_transactions[
len(logs_to_process) :
]
start_time = time.time()
try:
for i in range(n_retry_times + 1):
@ -3675,9 +3676,7 @@ async def update_spend( # noqa: PLR0915
# Check queue size with lock protection
async with prisma_client._spend_log_transactions_lock:
queue_size = len(prisma_client.spend_log_transactions)
verbose_proxy_logger.debug(
"Spend Logs transactions: {}".format(queue_size)
)
verbose_proxy_logger.debug("Spend Logs transactions: {}".format(queue_size))
# Process spend log transactions when called directly.
# This keeps backwards compatibility with the old behavior.
@ -3699,19 +3698,19 @@ async def update_spend_logs_job(
):
"""
Job to process spend_log_transactions queue.
This job is triggered based on queue size rather than time.
Processes spend log transactions when the queue reaches a threshold.
"""
n_retry_times = 3
# Check queue size with lock protection
async with prisma_client._spend_log_transactions_lock:
queue_size = len(prisma_client.spend_log_transactions)
if queue_size == 0:
return
await ProxyUpdateSpend.update_spend_logs(
n_retry_times=n_retry_times,
prisma_client=prisma_client,
@ -3728,7 +3727,7 @@ async def _monitor_spend_logs_queue(
"""
Background task that monitors the spend_log_transactions queue size
and triggers processing when the threshold is reached.
Args:
prisma_client: Prisma client instance
db_writer_client: Optional HTTP handler for external spend logs endpoint
@ -3738,23 +3737,23 @@ async def _monitor_spend_logs_queue(
SPEND_LOG_QUEUE_POLL_INTERVAL,
SPEND_LOG_QUEUE_SIZE_THRESHOLD,
)
threshold = SPEND_LOG_QUEUE_SIZE_THRESHOLD
base_interval = SPEND_LOG_QUEUE_POLL_INTERVAL
max_backoff = 30.0 # Maximum backoff interval in seconds
backoff_multiplier = 1.5 # Exponential backoff multiplier
current_interval = base_interval
verbose_proxy_logger.info(
f"Starting spend logs queue monitor (threshold: {threshold}, poll_interval: {base_interval}s)"
)
while True:
try:
# Check queue size with lock protection
async with prisma_client._spend_log_transactions_lock:
queue_size = len(prisma_client.spend_log_transactions)
if queue_size > 0:
if queue_size >= threshold:
verbose_proxy_logger.debug(
@ -3767,8 +3766,10 @@ async def _monitor_spend_logs_queue(
f"Spend logs queue size ({queue_size}) below threshold ({threshold}), processing with backoff"
)
# Exponential backoff when below threshold but still processing
current_interval = min(current_interval * backoff_multiplier, max_backoff)
current_interval = min(
current_interval * backoff_multiplier, max_backoff
)
await update_spend_logs_job(
prisma_client=prisma_client,
db_writer_client=db_writer_client,
@ -3776,8 +3777,10 @@ async def _monitor_spend_logs_queue(
)
else:
# Exponential backoff when no logs to process
current_interval = min(current_interval * backoff_multiplier, max_backoff)
current_interval = min(
current_interval * backoff_multiplier, max_backoff
)
await asyncio.sleep(current_interval)
except Exception as e:
verbose_proxy_logger.error(
@ -3788,7 +3791,6 @@ async def _monitor_spend_logs_queue(
await asyncio.sleep(current_interval)
def _raise_failed_update_spend_exception(
e: Exception, start_time: float, proxy_logging_obj: ProxyLogging
):

2
litellm/realtime_api/main.py
View file

@ -3,7 +3,7 @@
from typing import Any, Optional, cast
import litellm
from litellm import get_llm_provider
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler

10
litellm/responses/mcp/litellm_proxy_mcp_handler.py
View file

@ -205,7 +205,15 @@ class LiteLLM_Proxy_MCP_Handler:
else:
tool_name = getattr(mcp_tool, "name", None)
if tool_name and tool_name in allowed_tool_names:
if not tool_name:
continue
if tool_name in allowed_tool_names:
filtered_tools.append(mcp_tool)
continue
unprefixed_name, _ = split_server_prefix_from_name(tool_name)
if unprefixed_name in allowed_tool_names:
filtered_tools.append(mcp_tool)
return filtered_tools

26
litellm/responses/utils.py
View file

@ -26,7 +26,7 @@ from litellm.types.llms.openai import (
from litellm.types.responses.main import DecodedResponseId
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetails,
PromptTokensDetailsWrapper,
SpecialEnums,
Usage,
)
@ -431,7 +431,12 @@ class ResponseAPILoggingUtils:
def _transform_response_api_usage_to_chat_usage(
usage_input: Optional[Union[dict, ResponseAPIUsage]],
) -> Usage:
"""Tranforms the ResponseAPIUsage object to a Usage object"""
"""
Transforms ResponseAPIUsage or ImageUsage to a Usage object.
Both have the same spec with input_tokens, output_tokens, and
input_tokens_details (text_tokens, image_tokens).
"""
if usage_input is None:
return Usage(
prompt_tokens=0,
@ -445,18 +450,19 @@ class ResponseAPILoggingUtils:
)
prompt_tokens: int = response_api_usage.input_tokens or 0
completion_tokens: int = response_api_usage.output_tokens or 0
prompt_tokens_details: Optional[PromptTokensDetails] = None
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
if response_api_usage.input_tokens_details:
prompt_tokens_details = PromptTokensDetails(
cached_tokens=response_api_usage.input_tokens_details.cached_tokens,
audio_tokens=response_api_usage.input_tokens_details.audio_tokens,
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None),
audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None),
text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None),
image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None),
)
completion_tokens_details: Optional[CompletionTokensDetailsWrapper] = None
if response_api_usage.output_tokens_details:
output_tokens_details = getattr(response_api_usage, "output_tokens_details", None)
if output_tokens_details:
completion_tokens_details = CompletionTokensDetailsWrapper(
reasoning_tokens=getattr(
response_api_usage.output_tokens_details, "reasoning_tokens", None
)
reasoning_tokens=getattr(output_tokens_details, "reasoning_tokens", None)
)
chat_usage = Usage(

2
litellm/router_utils/pattern_match_deployments.py
View file

@ -7,7 +7,7 @@ import re
from re import Match
from typing import Dict, List, Optional, Tuple
from litellm import get_llm_provider
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm._logging import verbose_router_logger

33
litellm/types/llms/openai.py
View file

@ -1197,6 +1197,39 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject):
# Define private attributes using PrivateAttr
_hidden_params: dict = PrivateAttr(default_factory=dict)
@property
def output_text(self) -> str:
"""
Convenience property that aggregates all `output_text` items from the `output` list.
If no `output_text` content blocks exist, then an empty string is returned.
This matches the OpenAI SDK's Response.output_text behavior.
"""
texts: List[str] = []
for output_item in self.output:
# Handle both dict and object access patterns
if isinstance(output_item, dict):
item_type = output_item.get("type")
content = output_item.get("content", [])
else:
item_type = getattr(output_item, "type", None)
content = getattr(output_item, "content", [])
if item_type == "message":
for content_item in content:
if isinstance(content_item, dict):
content_type = content_item.get("type")
text = content_item.get("text", "")
else:
content_type = getattr(content_item, "type", None)
text = getattr(content_item, "text", "") or ""
if content_type == "output_text":
texts.append(text)
return "".join(texts)
class ResponsesAPIStreamEvents(str, Enum):
"""

3
litellm/types/utils.py
View file

@ -142,6 +142,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
] # only for vertex ai models
input_cost_per_query: Optional[float] # only for rerank models
input_cost_per_image: Optional[float] # only for vertex ai models
input_cost_per_image_token: Optional[float] # for gpt-image-1 and similar models
input_cost_per_audio_per_second: Optional[float] # only for vertex ai models
input_cost_per_video_per_second: Optional[float] # only for vertex ai models
input_cost_per_second: Optional[float] # for OpenAI Speech models
@ -1300,7 +1301,7 @@ class CacheCreationTokenDetails(BaseModel):
class PromptTokensDetailsWrapper(
PromptTokensDetails
): # wrapper for older openai versions
): # extends with image generation fields (text_tokens, image_tokens)
text_tokens: Optional[int] = None
"""Text tokens sent to the model."""

1095
litellm/utils.py
View file

File diff suppressed because it is too large Load diff

Some files were not shown because too many files have changed in this diff Show more