scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-14 23:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 102

Merge remote-tracking branch 'upstream/main' into litellm_feat_guardrail-log-actual-event-type-ProxyDBLogger

Browse source
This commit is contained in:
Yuta Saito 2026-01-05 05:47:59 +09:00
commit fd748b681b
277 changed files with 24390 additions and 4804 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

182
.circleci/config.yml
View file

@ -1980,6 +1980,7 @@ jobs:
- run: python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
- run: python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
- run: python ./tests/code_coverage_tests/check_fastuuid_usage.py
- run: python ./tests/code_coverage_tests/memory_test.py
- run: helm lint ./deploy/charts/litellm-helm
db_migration_disable_update_check:
@ -2008,10 +2009,13 @@ jobs:
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install apscheduler
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
name: Load Docker Database Image
command: |
docker build -t myapp . -f ./docker/Dockerfile.database
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
command: |
@ -2024,7 +2028,7 @@ jobs:
-v $(pwd)/litellm/proxy/example_config_yaml/bad_schema.prisma:/app/litellm/proxy/schema.prisma \
-v $(pwd)/litellm/proxy/example_config_yaml/disable_schema_update.yaml:/app/config.yaml \
--name my-app \
myapp:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000
- run:
@ -2276,9 +2280,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
command: |
@ -2313,7 +2321,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/oai_misc_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -2416,9 +2424,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@ -2451,7 +2463,7 @@ jobs:
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/otel_test_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_guardrail.py:/app/custom_guardrail.py \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -2502,7 +2514,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app-3 \
-v $(pwd)/litellm/proxy/example_config_yaml/enterprise_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
@ -2577,9 +2589,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@ -2603,7 +2619,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -2690,9 +2706,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container 1
# intentionally give bad redis credentials here
@ -2712,7 +2732,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -2733,7 +2753,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app-2 \
-v $(pwd)/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4001 \
--detailed_debug
@ -2826,9 +2846,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
# intentionally give bad redis credentials here
@ -2843,7 +2867,7 @@ jobs:
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/store_model_db_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -3058,10 +3082,13 @@ jobs:
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
# Run pytest and generate JUnit XML report
- attach_workspace:
at: ~/project
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
command: |
@ -3083,7 +3110,7 @@ jobs:
--name my-app \
-v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \
-v $(pwd)/litellm/proxy/example_config_yaml/custom_auth_basic.py:/app/custom_auth_basic.py \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
@ -3421,6 +3448,37 @@ jobs:
--coverage.reporter=html \
--coverage.reportsDirectory=coverage/html
build_docker_database_image:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- run:
name: Upgrade Docker
command: |
curl -fsSL https://get.docker.com | sh
docker version
- run:
name: Build Docker image
command: |
docker build \
-t litellm-docker-database:ci \
-f docker/Dockerfile.database .
- run:
name: Save Docker image to workspace root
command: |
docker save litellm-docker-database:ci | gzip > litellm-docker-database.tar.gz
- persist_to_workspace:
root: .
paths:
- litellm-docker-database.tar.gz
e2e_ui_testing:
machine:
image: ubuntu-2204:2023.10.1
@ -3432,54 +3490,18 @@ jobs:
- attach_workspace:
at: ~/project
- run:
name: Upgrade Docker to v24.x (API 1.44+)
name: Load Docker Database Image
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install Python 3.9
command: |
curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh --output miniconda.sh
bash miniconda.sh -b -p $HOME/miniconda
export PATH="$HOME/miniconda/bin:$PATH"
conda init bash
source ~/.bashrc
conda create -n myenv python=3.9 -y
conda activate myenv
python --version
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Install Dependencies
command: |
npm install -D @playwright/test
npm install @google-cloud/vertexai
pip install "pytest==7.3.1"
pip install "pytest-retry==1.6.3"
pip install "pytest-asyncio==0.21.1"
pip install aiohttp
pip install "openai==1.100.1"
python -m pip install --upgrade pip
pip install "pydantic==2.10.2"
pip install "pytest==7.3.1"
pip install "pytest-mock==3.12.0"
pip install "pytest-asyncio==0.21.1"
pip install "mypy==1.18.2"
pip install pyarrow
pip install numpydoc
pip install prisma
pip install fastapi
pip install jsonschema
pip install "httpx==0.24.1"
pip install "anyio==3.7.1"
pip install "asyncio==3.4.3"
- run:
name: Install Playwright Browsers
command: |
npx playwright install
- run:
name: Build Docker image
command: docker build -t my-app:latest -f ./docker/Dockerfile.database .
- run:
name: Run Docker container
command: |
@ -3491,9 +3513,9 @@ jobs:
-e UI_USERNAME="admin" \
-e UI_PASSWORD="gm" \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--name my-app \
--name litellm-docker-database \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
my-app:latest \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug
@ -3507,7 +3529,7 @@ jobs:
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start outputting logs
command: docker logs -f my-app
command: docker logs -f litellm-docker-database
background: true
- run:
name: Wait for app to be ready
@ -3515,7 +3537,10 @@ jobs:
- run:
name: Run Playwright Tests
command: |
npx playwright test e2e_ui_tests/ --reporter=html --output=test-results
npx playwright test \
--config ui/litellm-dashboard/e2e_tests/playwright.config.ts \
--reporter=html \
--output=test-results
no_output_timeout: 120m
- store_artifacts:
path: test-results
@ -3705,9 +3730,16 @@ workflows:
only:
- main
- /litellm_.*/
- build_docker_database_image:
filters:
branches:
only:
- main
- /litellm_.*/
- e2e_ui_testing:
requires:
- ui_build
- build_docker_database_image
filters:
branches:
only:
@ -3720,30 +3752,40 @@ workflows:
- main
- /litellm_.*/
- e2e_openai_endpoints:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_logging_guardrails_model_info_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_spend_accuracy_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_multi_instance_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_store_model_in_db_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
@ -3756,6 +3798,8 @@ workflows:
- main
- /litellm_.*/
- proxy_pass_through_endpoint_tests:
requires:
- build_docker_database_image
filters:
branches:
only:
@ -3894,6 +3938,8 @@ workflows:
- litellm_assistants_api_testing
- auth_ui_unit_tests
- db_migration_disable_update_check:
requires:
- build_docker_database_image
filters:
branches:
only:
@ -3973,4 +4019,4 @@ workflows:
- proxy_pass_through_endpoint_tests
- check_code_and_doc_quality
- publish_proxy_extras
- guardrails_testing
- guardrails_testing

7
.gitguardian.yaml
View file

@ -84,6 +84,10 @@ secret:
- name: Langfuse test credentials in test_completion
match: c39310f68cc3d3e22f7b298bb6353c4f45759adcc37080d8b7f4e535d3cfd7f4
# Test password "sk-1234" in e2e test fixtures - test fixture, not a real secret
- name: Test password in e2e test fixtures
match: ce32b547202e209ec1dd50107b64be4cfcf2eb15c3b4f8e9dc611ef747af634f
# === Preventive patterns for test keys (pattern-based) ===
# Test API keys (124 instances across 45 files)
@ -102,3 +106,6 @@ secret:
- name: Test API key patterns
match: test-api-key
- name: Short fake sk keys (1–9 digits only)
match: \bsk-\d{1,9}\b

4
.gitignore vendored
View file

@ -101,3 +101,7 @@ tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
litellm/proxy/_experimental/out/guardrails/index.html
scripts/test_vertex_ai_search.py
LAZY_LOADING_IMPROVEMENTS.md
**/test-results
**/playwright-report
**/*.storageState.json
**/coverage

1
ci_cd/security_scans.sh
View file

@ -128,6 +128,7 @@ run_grype_scans() {
"GHSA-5j98-mcp5-4vw2"
"CVE-2025-13836" # Python 3.13 HTTP response reading OOM/DoS - no fix available in base image
"CVE-2025-12084" # Python 3.13 xml.dom.minidom quadratic algorithm - no fix available in base image
"CVE-2025-60876" # BusyBox wget HTTP request splitting - no fix available in Chainguard Wolfi base image
)
# Build JSON array of allowlisted CVE IDs for jq

4
deploy/charts/litellm-helm/templates/deployment.yaml
View file

@ -182,6 +182,10 @@ spec:
{{- with .Values.volumeMounts }}
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.lifecycle }}
lifecycle:
{{- toYaml . | nindent 12 }}
{{- end }}
{{- with .Values.extraContainers }}
{{- toYaml . | nindent 8 }}
{{- end }}

24
deploy/charts/litellm-helm/tests/deployment_tests.yaml
View file

@ -136,4 +136,26 @@ tests:
path: spec.template.spec.containers[0].volumeMounts
content:
name: litellm-config
mountPath: /etc/litellm/
mountPath: /etc/litellm/
- it: should work with lifecycle hooks
template: deployment.yaml
set:
lifecycle:
preStop:
exec:
command:
- /bin/sh
- -c
- echo "Container stopping"
asserts:
- exists:
path: spec.template.spec.containers[0].lifecycle
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[0]
value: /bin/sh
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[1]
value: -c
- equal:
path: spec.template.spec.containers[0].lifecycle.preStop.exec.command[2]
value: echo "Container stopping"

16
docs/my-website/docs/mcp.md
View file

@ -110,6 +110,22 @@ For stdio MCP servers, select "Standard Input/Output (stdio)" as the transport t
<br/>
<br/>
### OAuth Configuration & Overrides
LiteLLM attempts [OAuth 2.0 Authorization Server Discovery](https://datatracker.ietf.org/doc/html/rfc8414) by default. When you create an MCP server in the UI and set `Authentication: OAuth`, LiteLLM will locate the provider metadata, dynamically register a client, and perform PKCE-based authorization without you providing any additional details.
**Customize the OAuth flow when needed:**
<Image
img={require('../img/mcp_oauth.png')}
style={{width: '80%', display: 'block', margin: '0'}}
/>
- **Provide explicit client credentials** – If the MCP provider does not offer dynamic client registration or you prefer to manage the client yourself, fill in `client_id`, `client_secret`, and the desired `scopes`.
- **Override discovery URLs** – In some environments, LiteLLM might not be able to reach the provider's metadata endpoints. Use the optional `authorization_url`, `token_url`, and `registration_url` fields to point LiteLLM directly to the correct endpoints.
<br/>
### Static Headers
Sometimes your MCP server needs specific headers on every request. Maybe it's an API key, maybe it's a custom header the server expects. Instead of configuring auth, you can just set them directly.

59
docs/my-website/docs/observability/generic_api.md
View file

@ -47,6 +47,7 @@ callback_settings:
| `endpoint` | string | Yes | HTTP endpoint to send logs to |
| `headers` | dict | No | Custom headers for the request |
| `event_types` | list | No | Filter events: `llm_api_success`, `llm_api_failure`. Defaults to all events. |
| `log_format` | string | No | Output format: `json_array` (default), `ndjson`, or `single`. Controls how logs are batched and sent. |
## Pre-configured Callbacks
@ -107,4 +108,62 @@ callback_settings:
flush_interval: 60 # seconds, default: 60
```
## Log Format Options
Control how logs are formatted and sent to your endpoint.
### JSON Array (Default)
```yaml
callback_settings:
my_api:
callback_type: generic_api
endpoint: https://your-endpoint.com
log_format: json_array # default if not specified
```
Sends all logs in a batch as a single JSON array `[{log1}, {log2}, ...]`. This is the default behavior and maintains backward compatibility.
**When to use**: Most HTTP endpoints expecting batched JSON data.
### NDJSON (Newline-Delimited JSON)
```yaml
callback_settings:
my_api:
callback_type: generic_api
endpoint: https://your-endpoint.com
log_format: ndjson
```
Sends logs as newline-delimited JSON (one record per line):
```
{log1}
{log2}
{log3}
```
**When to use**: Log aggregation services like Sumo Logic, Splunk, or Datadog that support field extraction on individual records.
**Benefits**:
- Each log is ingested as a separate message
- Field Extraction Rules work at ingest time
- Better parsing and querying performance
### Single
```yaml
callback_settings:
my_api:
callback_type: generic_api
endpoint: https://your-endpoint.com
log_format: single
```
Sends each log as an individual HTTP request in parallel when the batch is flushed.
**When to use**: Endpoints that expect individual records, or when you need maximum compatibility.
**Note**: This mode sends N HTTP requests per batch (more overhead). Consider using `ndjson` instead if your endpoint supports it.

4
docs/my-website/docs/observability/opentelemetry_integration.md
View file

@ -12,7 +12,9 @@ OpenTelemetry is a CNCF standard for observability. It connects to any observabi
From v1.81.0, the request/response will be set as attributes on the parent "Received Proxy Server Request" span by default. This allows you to see the request/response in the parent span in your observability tool.
To use the older behavior with nested "litellm_request" spans, set the following environment variable:
**Note:** When making multiple LLM calls within an external OTEL span context, the last call's attributes will overwrite previous calls' attributes on the parent span.
To use the older behavior with nested "litellm_request" spans (which creates separate spans for each call), set the following environment variable:
```shell
USE_OTEL_LITELLM_REQUEST_SPAN=true

45
docs/my-website/docs/observability/sumologic_integration.md
View file

@ -148,6 +148,51 @@ Example payload:
## Advanced Configuration
### Log Format
The Sumo Logic integration uses **NDJSON (newline-delimited JSON)** format by default. This format is optimal for Sumo Logic's parsing capabilities and allows Field Extraction Rules to work at ingest time.
#### NDJSON Format
Each log entry is sent as a separate line in the HTTP request:
```
{"id":"chatcmpl-1","model":"gpt-3.5-turbo","response_cost":0.0001,...}
{"id":"chatcmpl-2","model":"gpt-4","response_cost":0.0003,...}
{"id":"chatcmpl-3","model":"gpt-3.5-turbo","response_cost":0.0001,...}
```
#### Benefits for Field Extraction Rules (FERs)
With NDJSON format, you can create Field Extraction Rules directly:
```
_sourceCategory=litellm/logs
| json field=_raw "model", "response_cost", "user" as model, cost, user
```
**Before NDJSON** (with JSON array format):
- Required `parse regex ... multi` workaround
- FERs couldn't parse at ingest time
- Query-time parsing impacted dashboard performance
**After NDJSON**:
- ✅ FERs parse fields at ingest time
- ✅ No query-time workarounds needed
- ✅ Better dashboard performance
- ✅ Simpler query syntax
#### Changing the Log Format (Advanced)
If you need to change the log format (not recommended for Sumo Logic):
```yaml
callback_settings:
sumologic:
callback_type: generic_api
callback_name: sumologic
log_format: json_array # Override to use JSON array instead
```
### Batching Settings
Control how LiteLLM batches logs before sending to Sumo Logic:

4
docs/my-website/docs/providers/anthropic.md
View file

@ -444,7 +444,7 @@ Here's what a sample Raw Request from LiteLLM for Anthropic Context Caching look
POST Request Sent from LiteLLM:
curl -X POST \
https://api.anthropic.com/v1/messages \
-H 'accept: application/json' -H 'anthropic-version: 2023-06-01' -H 'content-type: application/json' -H 'x-api-key: sk-...' -H 'anthropic-beta: prompt-caching-2024-07-31' \
-H 'accept: application/json' -H 'anthropic-version: 2023-06-01' -H 'content-type: application/json' -H 'x-api-key: sk-...' \
-d '{'model': 'claude-3-5-sonnet-20240620', [
{
"role": "user",
@ -472,6 +472,8 @@ https://api.anthropic.com/v1/messages \
"max_tokens": 10
}'
```
**Note:** Anthropic no longer requires the `anthropic-beta: prompt-caching-2024-07-31` header. Prompt caching now works automatically when you use `cache_control` in your messages.
:::
### Caching - Large Context Caching

47
docs/my-website/docs/providers/bedrock.md
View file

@ -2208,6 +2208,53 @@ response = completion(
| `aws_role_name` | `RoleArn` | The Amazon Resource Name (ARN) of the role to assume | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
| `aws_session_name` | `RoleSessionName` | An identifier for the assumed role session | [AssumeRole API](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts.html#STS.Client.assume_role) |
### IAM Roles Anywhere (On-Premise / External Workloads)
[IAM Roles Anywhere](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/introduction.html) extends IAM roles to workloads **outside of AWS** (on-premise servers, edge devices, other clouds). It uses the same STS mechanism as regular IAM roles but authenticates via X.509 certificates instead of AWS credentials.
**Setup**: Configure the [AWS Signing Helper](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/credential-helper.html) as a credential process in `~/.aws/config`:
```ini
[profile litellm-roles-anywhere]
credential_process = aws_signing_helper credential-process \
--certificate /path/to/certificate.pem \
--private-key /path/to/private-key.pem \
--trust-anchor-arn arn:aws:rolesanywhere:us-east-1:123456789012:trust-anchor/abc123 \
--profile-arn arn:aws:rolesanywhere:us-east-1:123456789012:profile/def456 \
--role-arn arn:aws:iam::123456789012:role/MyBedrockRole
```
**Usage**: Reference the profile in LiteLLM:
<Tabs>
<TabItem value="sdk" label="SDK">
```python
from litellm import completion
response = completion(
model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
messages=[{"role": "user", "content": "Hello!"}],
aws_profile_name="litellm-roles-anywhere",
)
```
</TabItem>
<TabItem value="proxy" label="PROXY">
```yaml
model_list:
- model_name: bedrock-claude
litellm_params:
model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
aws_profile_name: "litellm-roles-anywhere"
```
</TabItem>
</Tabs>
See the [IAM Roles Anywhere Getting Started Guide](https://docs.aws.amazon.com/rolesanywhere/latest/userguide/getting-started.html) for trust anchor and profile setup.
Make the bedrock completion call

6
docs/my-website/docs/providers/bedrock_agentcore.md
View file

@ -11,6 +11,12 @@ Call Bedrock AgentCore in the OpenAI Request/Response format.
| Provider Route on LiteLLM | `bedrock/agentcore/{AGENT_RUNTIME_ARN}` |
| Provider Doc | [AWS Bedrock AgentCore ↗](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgentRuntime.html) |
:::info
This documentation is for **AgentCore Agents** (agent runtimes). If you want to use AgentCore MCP servers, add them as you would any other MCP server. See the [MCP documentation](https://docs.litellm.ai/docs/mcp) for details.
:::
## Quick Start
### Model Format to LiteLLM

36
docs/my-website/docs/providers/zai.md
View file

@ -19,7 +19,7 @@ import os
os.environ['ZAI_API_KEY'] = ""
response = completion(
model="zai/glm-4.6",
model="zai/glm-4.7",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@ -34,7 +34,7 @@ import os
os.environ['ZAI_API_KEY'] = ""
response = completion(
model="zai/glm-4.6",
model="zai/glm-4.7",
messages=[
{"role": "user", "content": "hello from litellm"}
],
@ -51,7 +51,8 @@ We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending complet
| Model Name | Function Call | Notes |
|------------|---------------|-------|
| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | Latest flagship model, 200K context |
| glm-4.7 | `completion(model="zai/glm-4.7", messages)` | **Latest flagship**, 200K context, **Reasoning** |
| glm-4.6 | `completion(model="zai/glm-4.6", messages)` | 200K context |
| glm-4.5 | `completion(model="zai/glm-4.5", messages)` | 128K context |
| glm-4.5v | `completion(model="zai/glm-4.5v", messages)` | Vision model |
| glm-4.5-x | `completion(model="zai/glm-4.5-x", messages)` | Premium tier |
@ -62,16 +63,17 @@ We support ALL Z.AI GLM models, just set `zai/` as a prefix when sending complet
## Model Pricing
| Model | Input ($/1M tokens) | Output ($/1M tokens) | Context Window |
|-------|---------------------|----------------------|----------------|
| glm-4.6 | $0.60 | $2.20 | 200K |
| glm-4.5 | $0.60 | $2.20 | 128K |
| glm-4.5v | $0.60 | $1.80 | 128K |
| glm-4.5-x | $2.20 | $8.90 | 128K |
| glm-4.5-air | $0.20 | $1.10 | 128K |
| glm-4.5-airx | $1.10 | $4.50 | 128K |
| glm-4-32b-0414-128k | $0.10 | $0.10 | 128K |
| glm-4.5-flash | **FREE** | **FREE** | 128K |
| Model | Input ($/1M tokens) | Output ($/1M tokens) | Cached Input ($/1M tokens) | Context Window |
|-------|---------------------|----------------------|---------------------------|----------------|
| glm-4.7 | $0.60 | $2.20 | $0.11 | 200K |
| glm-4.6 | $0.60 | $2.20 | - | 200K |
| glm-4.5 | $0.60 | $2.20 | - | 128K |
| glm-4.5v | $0.60 | $1.80 | - | 128K |
| glm-4.5-x | $2.20 | $8.90 | - | 128K |
| glm-4.5-air | $0.20 | $1.10 | - | 128K |
| glm-4.5-airx | $1.10 | $4.50 | - | 128K |
| glm-4-32b-0414-128k | $0.10 | $0.10 | - | 128K |
| glm-4.5-flash | **FREE** | **FREE** | - | 128K |
## Using with LiteLLM Proxy
@ -84,7 +86,7 @@ import os
os.environ['ZAI_API_KEY'] = ""
response = completion(
model="zai/glm-4.6",
model="zai/glm-4.7",
messages=[{"role": "user", "content": "Hello, how are you?"}],
)
@ -98,9 +100,9 @@ print(response.choices[0].message.content)
```yaml
model_list:
- model_name: glm-4.6
- model_name: glm-4.7
litellm_params:
model: zai/glm-4.6
model: zai/glm-4.7
api_key: os.environ/ZAI_API_KEY
- model_name: glm-4.5-flash # Free tier
litellm_params:
@ -121,7 +123,7 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer sk-1234' \
-d '{
"model": "glm-4.6",
"model": "glm-4.7",
"messages": [
{
"role": "user",

4
docs/my-website/docs/proxy/config_settings.md
View file

@ -464,6 +464,9 @@ router_settings:
| DATABASE_USER | Username for database connection
| DATABASE_USERNAME | Alias for database user
| DATABRICKS_API_BASE | Base URL for Databricks API
| DATABRICKS_CLIENT_ID | Client ID for Databricks OAuth M2M authentication (Service Principal application ID)
| DATABRICKS_CLIENT_SECRET | Client secret for Databricks OAuth M2M authentication
| DATABRICKS_USER_AGENT | Custom user agent string for Databricks API requests. Used for partner telemetry attribution
| DAYS_IN_A_MONTH | Days in a month for calculation purposes. Default is 28
| DAYS_IN_A_WEEK | Days in a week for calculation purposes. Default is 7
| DAYS_IN_A_YEAR | Days in a year for calculation purposes. Default is 365
@ -708,6 +711,7 @@ router_settings:
| LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
| LITELLM_TOKEN | Access token for LiteLLM integration
| LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LOGFIRE_TOKEN | Token for Logfire logging service

2
docs/my-website/docs/proxy/configs.md
View file

@ -116,7 +116,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \
"role": "user",
"content": "what llm are you"
}
],
]
}
'
```

239
docs/my-website/docs/proxy/token_auth.md
View file

@ -114,6 +114,189 @@ Set `JWT_PUBLIC_KEY_URL` in your environment to a comma-separated list of URLs f
export JWT_PUBLIC_KEY_URL="https://demo.duendesoftware.com/.well-known/openid-configuration/jwks,https://accounts.google.com/.well-known/openid-configuration/jwks"
```
### Kubernetes ServiceAccount Authentication
Use Kubernetes ServiceAccount tokens to authenticate workloads running in your cluster. This is useful when you want pods to authenticate to LiteLLM using their native Kubernetes identity.
#### Prerequisites
1. Your Kubernetes cluster must have ServiceAccount token projection enabled (default in Kubernetes 1.20+)
2. Your cluster's OIDC issuer must be accessible (for EKS, GKE, AKS this is automatic)
#### Step 1: Configure the OIDC Discovery URL
Set `JWT_PUBLIC_KEY_URL` to your cluster's OIDC discovery endpoint:
<Tabs>
<TabItem value="eks" label="Amazon EKS">
```bash
# Get your EKS OIDC issuer URL
aws eks describe-cluster --name <cluster-name> --query "cluster.identity.oidc.issuer" --output text
# Set the JWKS URL (append /keys to the issuer URL)
export JWT_PUBLIC_KEY_URL="https://oidc.eks.<region>.amazonaws.com/id/<id>/keys"
```
</TabItem>
<TabItem value="gke" label="Google GKE">
```bash
# GKE uses Google's OIDC provider
export JWT_PUBLIC_KEY_URL="https://container.googleapis.com/v1/projects/<project>/locations/<location>/clusters/<cluster>/jwks"
```
</TabItem>
<TabItem value="aks" label="Azure AKS">
```bash
# Get your AKS OIDC issuer URL
az aks show --name <cluster-name> --resource-group <resource-group> --query "oidcIssuerProfile.issuerUrl" -o tsv
# Set the JWKS URL
export JWT_PUBLIC_KEY_URL="<issuer-url>/openid/v1/jwks"
```
</TabItem>
<TabItem value="self-managed" label="Self-Managed">
```bash
# For self-managed clusters, check your API server's --service-account-issuer flag
# The JWKS endpoint is typically at:
export JWT_PUBLIC_KEY_URL="https://<api-server>/openid/v1/jwks"
```
</TabItem>
</Tabs>
#### Step 2: Configure LiteLLM
Configure LiteLLM to extract identity information from Kubernetes ServiceAccount tokens:
```yaml
general_settings:
enable_jwt_auth: True
litellm_jwtauth:
# Use namespace as team identifier (resolves via team_alias in DB)
team_alias_jwt_field: "kubernetes\.io.namespace"
```
#### Step 3: Create ServiceAccount and Configure Pod
Create a ServiceAccount with an associated secret and configure your pod to use the token:
```yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: my-llm-client
namespace: my-app
---
apiVersion: v1
kind: Secret
metadata:
name: my-llm-client-token
namespace: my-app
annotations:
kubernetes.io/service-account.name: my-llm-client
type: kubernetes.io/service-account-token
---
apiVersion: v1
kind: Pod
metadata:
name: llm-client-pod
namespace: my-app
spec:
serviceAccountName: my-llm-client
containers:
- name: app
image: my-app:latest
env:
- name: LITELLM_TOKEN
valueFrom:
secretKeyRef:
name: my-llm-client-token
key: token
```
Set the expected audience in LiteLLM:
```bash
export JWT_AUDIENCE="https://kubernetes.default.svc"
```
#### Step 4: Create Team for Namespace
Create a team in LiteLLM that matches the namespace (using `team_alias`):
```bash
curl -X POST 'http://0.0.0.0:4000/team/new' \
-H 'Authorization: Bearer <PROXY_MASTER_KEY>' \
-H 'Content-Type: application/json' \
-d '{
"team_alias": "my-app",
"team_id": "my-app",
"models": ["gpt-4", "claude-sonnet-4-20250514"]
}'
```
#### Step 5: Use the Token
From within the pod, the token is available in the `LITELLM_TOKEN` environment variable:
```bash
# Make a request to LiteLLM using the env var
curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $LITELLM_TOKEN" \
-d '{
"model": "gpt-4",
"messages": [{"role": "user", "content": "Hello!"}]
}'
```
#### Example: ServiceAccount Token Structure
A Kubernetes ServiceAccount token looks like this:
```json
{
"aud": ["litellm-proxy"],
"exp": 1234567890,
"iat": 1234567890,
"iss": "https://oidc.eks.us-west-2.amazonaws.com/id/EXAMPLE",
"kubernetes.io": {
"namespace": "my-app",
"pod": {
"name": "llm-client-pod",
"uid": "pod-uid"
},
"serviceaccount": {
"name": "my-llm-client",
"uid": "sa-uid"
}
},
"nbf": 1234567890,
"sub": "system:serviceaccount:my-app:my-llm-client"
}
```
#### Advanced: Map Namespace to Team Using Name Resolution
Use the `team_alias_jwt_field` to automatically resolve namespaces to teams:
```yaml
general_settings:
enable_jwt_auth: True
litellm_jwtauth:
user_id_jwt_field: "sub"
# Map the namespace to team_alias in the database
team_alias_jwt_field: "kubernetes\.io.namespace"
user_id_upsert: true
```
This way, pods in namespace `production` automatically get associated with the team that has `team_alias: production`.
### Set Accepted JWT Scope Names
Change the string in JWT 'scopes', that litellm evaluates to see if a user has admin access.
@ -183,6 +366,62 @@ litellm_jwtauth:
Now litellm will automatically update the spend for the user/team/org in the db for each call.
### Resolve by Name (Alias) Instead of ID
Sometimes your JWT token contains human-readable names instead of database IDs. LiteLLM can resolve these names to IDs by looking them up in the database.
**Use Case:** Your IDP provides team/org names in the JWT, but LiteLLM needs the actual database IDs for spend tracking and access control.
```yaml
general_settings:
master_key: sk-1234
enable_jwt_auth: True
litellm_jwtauth:
# Name-based fields (resolved via database lookup)
team_alias_jwt_field: "team_alias" # Resolves team by team_alias in DB
org_alias_jwt_field: "org_alias" # Resolves org by organization_alias in DB
```
**Expected JWT:**
```json
{
"sub": "user-123",
"team_alias": "engineering-team",
"org_alias": "acme-corp"
}
```
**How It Works:**
1. LiteLLM extracts the name from the configured JWT field
2. Looks up the entity in the database by its alias field:
- Teams: `team_alias` column in `LiteLLM_TeamTable`
- Organizations: `organization_alias` column in `LiteLLM_OrganizationTable`
3. Uses the resolved ID for spend tracking and access control
**Precedence:** ID fields always take precedence over name fields. If both `team_id_jwt_field` and `team_alias_jwt_field` are configured and both values exist in the JWT, the ID will be used.
```yaml
# Example: ID takes precedence
litellm_jwtauth:
team_id_jwt_field: "team_id" # Used if present in JWT
team_alias_jwt_field: "team_alias" # Fallback if team_id not present
```
**Nested Fields:** Name fields also support dot notation for nested claims:
```yaml
litellm_jwtauth:
team_alias_jwt_field: "organization.team.name"
org_alias_jwt_field: "company.name"
```
**Important Notes:**
- The entity (team/org) must already exist in the database with the matching alias
- Aliases should be unique - if multiple entities share the same alias, an error will be returned
- Name resolution adds a database lookup, so using IDs directly is slightly more performant
### JWT Scopes
Here's what scopes on JWT-Auth tokens look like

146
docs/my-website/package-lock.json generated
View file

@ -8904,23 +8904,23 @@
"license": "ISC"
},
"node_modules/body-parser": {
"version": "1.20.3",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz",
"integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==",
"version": "1.20.4",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz",
"integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==",
"license": "MIT",
"dependencies": {
"bytes": "3.1.2",
"bytes": "~3.1.2",
"content-type": "~1.0.5",
"debug": "2.6.9",
"depd": "2.0.0",
"destroy": "1.2.0",
"http-errors": "2.0.0",
"iconv-lite": "0.4.24",
"on-finished": "2.4.1",
"qs": "6.13.0",
"raw-body": "2.5.2",
"destroy": "~1.2.0",
"http-errors": "~2.0.1",
"iconv-lite": "~0.4.24",
"on-finished": "~2.4.1",
"qs": "~6.14.0",
"raw-body": "~2.5.3",
"type-is": "~1.6.18",
"unpipe": "1.0.0"
"unpipe": "~1.0.0"
},
"engines": {
"node": ">= 0.8",
@ -8945,6 +8945,26 @@
"ms": "2.0.0"
}
},
"node_modules/body-parser/node_modules/http-errors": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
"integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
"license": "MIT",
"dependencies": {
"depd": "~2.0.0",
"inherits": "~2.0.4",
"setprototypeof": "~1.2.0",
"statuses": "~2.0.2",
"toidentifier": "~1.0.1"
},
"engines": {
"node": ">= 0.8"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/body-parser/node_modules/iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@ -8957,12 +8977,27 @@
"node": ">=0.10.0"
}
},
"node_modules/body-parser/node_modules/inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"license": "ISC"
},
"node_modules/body-parser/node_modules/ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
"license": "MIT"
},
"node_modules/body-parser/node_modules/statuses": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
"integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/bonjour-service": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/bonjour-service/-/bonjour-service-1.3.0.tgz",
@ -11873,39 +11908,39 @@
}
},
"node_modules/express": {
"version": "4.21.2",
"resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz",
"integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==",
"version": "4.22.1",
"resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz",
"integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==",
"license": "MIT",
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
"body-parser": "1.20.3",
"content-disposition": "0.5.4",
"body-parser": "~1.20.3",
"content-disposition": "~0.5.4",
"content-type": "~1.0.4",
"cookie": "0.7.1",
"cookie-signature": "1.0.6",
"cookie": "~0.7.1",
"cookie-signature": "~1.0.6",
"debug": "2.6.9",
"depd": "2.0.0",
"encodeurl": "~2.0.0",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
"finalhandler": "1.3.1",
"fresh": "0.5.2",
"http-errors": "2.0.0",
"finalhandler": "~1.3.1",
"fresh": "~0.5.2",
"http-errors": "~2.0.0",
"merge-descriptors": "1.0.3",
"methods": "~1.1.2",
"on-finished": "2.4.1",
"on-finished": "~2.4.1",
"parseurl": "~1.3.3",
"path-to-regexp": "0.1.12",
"path-to-regexp": "~0.1.12",
"proxy-addr": "~2.0.7",
"qs": "6.13.0",
"qs": "~6.14.0",
"range-parser": "~1.2.1",
"safe-buffer": "5.2.1",
"send": "0.19.0",
"serve-static": "1.16.2",
"send": "~0.19.0",
"serve-static": "~1.16.2",
"setprototypeof": "1.2.0",
"statuses": "2.0.1",
"statuses": "~2.0.1",
"type-is": "~1.6.18",
"utils-merge": "1.0.1",
"vary": "~1.1.2"
@ -19281,12 +19316,12 @@
}
},
"node_modules/qs": {
"version": "6.13.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
"integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
"version": "6.14.1",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz",
"integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==",
"license": "BSD-3-Clause",
"dependencies": {
"side-channel": "^1.0.6"
"side-channel": "^1.1.0"
},
"engines": {
"node": ">=0.6"
@ -19362,15 +19397,15 @@
}
},
"node_modules/raw-body": {
"version": "2.5.2",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz",
"integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==",
"version": "2.5.3",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz",
"integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==",
"license": "MIT",
"dependencies": {
"bytes": "3.1.2",
"http-errors": "2.0.0",
"iconv-lite": "0.4.24",
"unpipe": "1.0.0"
"bytes": "~3.1.2",
"http-errors": "~2.0.1",
"iconv-lite": "~0.4.24",
"unpipe": "~1.0.0"
},
"engines": {
"node": ">= 0.8"
@ -19385,6 +19420,26 @@
"node": ">= 0.8"
}
},
"node_modules/raw-body/node_modules/http-errors": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
"integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==",
"license": "MIT",
"dependencies": {
"depd": "~2.0.0",
"inherits": "~2.0.4",
"setprototypeof": "~1.2.0",
"statuses": "~2.0.2",
"toidentifier": "~1.0.1"
},
"engines": {
"node": ">= 0.8"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/express"
}
},
"node_modules/raw-body/node_modules/iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@ -19397,6 +19452,21 @@
"node": ">=0.10.0"
}
},
"node_modules/raw-body/node_modules/inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"license": "ISC"
},
"node_modules/raw-body/node_modules/statuses": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
"integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==",
"license": "MIT",
"engines": {
"node": ">= 0.8"
}
},
"node_modules/rc": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",

88
docs/my-website/src/data/adopters/README.md Normal file
View file

@ -0,0 +1,88 @@
# LiteLLM Adopters
This directory contains data for organizations that use LiteLLM in production.
## Adding Your Organization
We've made it super easy to add your organization! Just follow the steps below.
### Quick Add (Recommended)
**[Edit adopters.json on GitHub →](https://github.com/BerriAI/litellm/edit/main/docs/my-website/src/data/adopters/adopters.json)**
This will open the GitHub editor in your browser where you can:
1. Add your organization's entry to the JSON array
2. Commit your changes
3. GitHub will automatically create a pull request for you!
No need to clone the repository or set up a development environment.
### JSON Format
Add your organization to the array in `adopters.json`:
```json
{
"name": "Your Organization Name",
"logoUrl": "https://yoursite.com/logo.svg",
"url": "https://yourcompany.com",
"description": "Brief description of how you use LiteLLM (shown on hover)"
}
```
### Fields
- **`name`** (required): Your organization's display name
- **`logoUrl`** (required): URL to your logo - can be either:
- External URL: `https://yoursite.com/logo.svg` (easiest!)
- Local path: `/img/adopters/your-logo.svg` (requires uploading logo file)
- **`url`** (optional): Your organization's website (makes the logo clickable)
- **`description`** (optional): Brief description shown when users hover over your logo
### Logo Options
#### Option 1: External URL (Easiest)
Simply provide a direct link to your logo hosted anywhere:
```json
"logoUrl": "https://yourcompany.com/assets/logo.svg"
```
#### Option 2: Local Logo (Better Performance)
If you prefer to host the logo locally:
1. Add your logo to `docs/my-website/static/img/adopters/your-company.svg`
2. Reference it as: `"logoUrl": "/img/adopters/your-company.svg"`
**Logo Specifications:**
- **Format**: SVG preferred (PNG also acceptable)
- **Dimensions**: 240x160px or similar 3:2 ratio recommended
- **Background**: Transparent or white background works best
### Example
```json
{
"name": "Acme Corporation",
"logoUrl": "https://acme.com/logo.svg",
"url": "https://acme.com",
"description": "Using LiteLLM to route requests across 50+ LLM providers"
}
```
### Display Order
Adopters are displayed alphabetically by organization name, so your position will be determined automatically.
### Need Help?
If you have questions about adding your organization:
- Ask in [GitHub Discussions](https://github.com/BerriAI/litellm/discussions)
- Join our [Discord community](https://discord.com/invite/wuPM9dRgDw)
Thank you for supporting LiteLLM! 🚅

8
docs/my-website/src/data/adopters/adopters.json Normal file
View file

@ -0,0 +1,8 @@
[
{
"name": "Your Logo Here",
"logoUrl": "/img/adopters/placeholder-company.svg",
"description": "Add your organization to show support for LiteLLM",
"url": "https://github.com/BerriAI/litellm/edit/main/docs/my-website/src/data/adopters/adopters.json"
}
]

23
docs/my-website/src/data/adopters/index.js Normal file
View file

@ -0,0 +1,23 @@
import adoptersData from './adopters.json';
/**
* @typedef {Object} Adopter
* @property {string} name - The organization's display name
* @property {string} logoUrl - URL to the organization's logo
* @property {string} [url] - The organization's website URL
* @property {string} [description] - Brief description shown on hover
*/
/**
* List of organizations using LiteLLM
* @type {Adopter[]}
*/
export const adopters = adoptersData;
/**
* Adopters sorted alphabetically by name
* @type {Adopter[]}
*/
export const sortedAdopters = [...adopters].sort((a, b) =>
a.name.localeCompare(b.name)
);

8
docs/my-website/static/img/adopters/placeholder-company.svg Normal file
View file

@ -0,0 +1,8 @@
<svg width="240" height="160" viewBox="0 0 240 160" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect width="240" height="160" rx="8" fill="#f8fafc"/>
<rect x="1" y="1" width="238" height="158" rx="7" stroke="#e2e8f0" stroke-width="2" stroke-dasharray="8 4"/>
<circle cx="120" cy="60" r="24" fill="#e2e8f0"/>
<path d="M120 48v24M108 60h24" stroke="#94a3b8" stroke-width="3" stroke-linecap="round"/>
<text x="120" y="110" text-anchor="middle" fill="#64748b" font-family="system-ui, -apple-system, sans-serif" font-size="14" font-weight="500">Add Your Logo</text>
<text x="120" y="130" text-anchor="middle" fill="#94a3b8" font-family="system-ui, -apple-system, sans-serif" font-size="11">Click to contribute</text>
</svg>
Side by side

After

Width:  |  Height:  |  Size: 736 B

5
litellm-proxy-extras/litellm_proxy_extras/migrations/20260102131258_add_metadata_urls_to_mcp_servers/migration.sql Normal file
View file

@ -0,0 +1,5 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "authorization_url" TEXT,
ADD COLUMN "registration_url" TEXT,
ADD COLUMN "token_url" TEXT;

3
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
View file

@ -208,6 +208,9 @@ model LiteLLM_MCPServerTable {
command String?
args String[] @default([])
env Json? @default("{}")
authorization_url String?
token_url String?
registration_url String?
}
// Generate Tokens for Proxy

24
litellm/__init__.py
View file

@ -553,6 +553,8 @@ docker_model_runner_models: Set = set()
amazon_nova_models: Set = set()
stability_models: Set = set()
github_copilot_models: Set = set()
minimax_models: Set = set()
aws_polly_models: Set = set()
def is_bedrock_pricing_only_model(key: str) -> bool:
@ -801,6 +803,10 @@ def add_known_models():
stability_models.add(key)
elif value.get("litellm_provider") == "github_copilot":
github_copilot_models.add(key)
elif value.get("litellm_provider") == "minimax":
minimax_models.add(key)
elif value.get("litellm_provider") == "aws_polly":
aws_polly_models.add(key)
add_known_models()
@ -1005,6 +1011,8 @@ models_by_provider: dict = {
"amazon_nova": amazon_nova_models,
"stability": stability_models,
"github_copilot": github_copilot_models,
"minimax": minimax_models,
"aws_polly": aws_polly_models,
}
# mapping for those models which have larger equivalents
@ -1049,8 +1057,8 @@ openai_image_generation_models = ["dall-e-2", "dall-e-3"]
openai_video_generation_models = ["sora-2"]
# timeout is lazy-loaded via __getattr__
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.core_helpers import remove_index_from_tool_calls
# get_llm_provider is lazy-loaded via __getattr__
# remove_index_from_tool_calls is lazy-loaded via __getattr__
# Import KeyManagementSettings here (before utils import) because _key_management_settings
# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils)
@ -1499,6 +1507,7 @@ if TYPE_CHECKING:
get_first_chars_messages: Callable[..., str]
get_provider_fields: Callable[..., List]
get_valid_models: Callable[..., list]
remove_index_from_tool_calls: Callable[..., None]
# Response types - truly lazy loaded only (not in main.py or elsewhere)
ModelResponseListIterator: Type[Any]
@ -1650,6 +1659,17 @@ def __getattr__(name: str) -> Any:
LoggingCallbackManager = __getattr__("LoggingCallbackManager")
_globals["logging_callback_manager"] = LoggingCallbackManager()
return _globals["logging_callback_manager"]
# Lazy load _service_logger module
if name == "_service_logger":
from ._lazy_imports import _get_litellm_globals
_globals = _get_litellm_globals()
# Check if already cached
if "_service_logger" not in _globals:
# Import the module lazily
import litellm._service_logger
_globals["_service_logger"] = litellm._service_logger
return _globals["_service_logger"]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

9
litellm/_lazy_imports.py
View file

@ -34,6 +34,7 @@ from ._lazy_imports_registry import (
DOTPROMPT_NAMES,
LLM_CONFIG_NAMES,
TYPES_NAMES,
LLM_PROVIDER_LOGIC_NAMES,
# Import maps
_UTILS_IMPORT_MAP,
_COST_CALCULATOR_IMPORT_MAP,
@ -45,6 +46,7 @@ from ._lazy_imports_registry import (
_DOTPROMPT_IMPORT_MAP,
_TYPES_IMPORT_MAP,
_LLM_CONFIGS_IMPORT_MAP,
_LLM_PROVIDER_LOGIC_IMPORT_MAP,
)
@ -181,6 +183,8 @@ def _get_lazy_import_registry() -> dict[str, Callable[[str], Any]]:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_configs
for name in TYPES_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_types
for name in LLM_PROVIDER_LOGIC_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_provider_logic
return _LAZY_IMPORT_REGISTRY
@ -297,6 +301,11 @@ def _lazy_import_litellm_logging(name: str) -> Any:
"""Handler for litellm_logging module (Logging, modify_integration)"""
return _generic_lazy_import(name, _LITELLM_LOGGING_IMPORT_MAP, "Litellm logging")
def _lazy_import_llm_provider_logic(name: str) -> Any:
"""Handler for LLM provider logic functions (get_llm_provider, etc.)"""
return _generic_lazy_import(name, _LLM_PROVIDER_LOGIC_IMPORT_MAP, "LLM provider logic")
# ============================================================================
# SPECIAL HANDLERS
# ============================================================================

16
litellm/_lazy_imports_registry.py
View file

@ -32,6 +32,7 @@ UTILS_NAMES = (
"ModelResponse", "ModelResponseStream", "EmbeddingResponse", "ImageResponse",
"TranscriptionResponse", "TextCompletionResponse", "get_provider_fields",
"ModelResponseListIterator", "get_valid_models", "timeout",
"get_llm_provider", "remove_index_from_tool_calls",
)
# Token counter names that support lazy loading via _lazy_import_token_counter
@ -287,6 +288,12 @@ TYPES_NAMES = (
# is accessed during import time in secret_managers/main.py
)
# LLM provider logic names that support lazy loading via _lazy_import_llm_provider_logic
LLM_PROVIDER_LOGIC_NAMES = (
"get_llm_provider",
"remove_index_from_tool_calls",
)
# Import maps for registry pattern - reduces repetition
_UTILS_IMPORT_MAP = {
"exception_type": (".utils", "exception_type"),
@ -330,6 +337,8 @@ _UTILS_IMPORT_MAP = {
"ModelResponseListIterator": (".utils", "ModelResponseListIterator"),
"get_valid_models": (".utils", "get_valid_models"),
"timeout": (".timeout", "timeout"),
"get_llm_provider": ("litellm.litellm_core_utils.get_llm_provider_logic", "get_llm_provider"),
"remove_index_from_tool_calls": ("litellm.litellm_core_utils.core_helpers", "remove_index_from_tool_calls"),
}
_COST_CALCULATOR_IMPORT_MAP = {
@ -386,6 +395,11 @@ _TYPES_IMPORT_MAP = {
"LoggingCallbackManager": ("litellm.litellm_core_utils.logging_callback_manager", "LoggingCallbackManager"),
}
_LLM_PROVIDER_LOGIC_IMPORT_MAP = {
"get_llm_provider": ("litellm.litellm_core_utils.get_llm_provider_logic", "get_llm_provider"),
"remove_index_from_tool_calls": ("litellm.litellm_core_utils.core_helpers", "remove_index_from_tool_calls"),
}
_LLM_CONFIGS_IMPORT_MAP = {
"AmazonConverseConfig": (".llms.bedrock.chat.converse_transformation", "AmazonConverseConfig"),
"OpenAILikeChatConfig": (".llms.openai_like.chat.handler", "OpenAILikeChatConfig"),
@ -587,6 +601,7 @@ __all__ = [
"DOTPROMPT_NAMES",
"LLM_CONFIG_NAMES",
"TYPES_NAMES",
"LLM_PROVIDER_LOGIC_NAMES",
# Import maps
"_UTILS_IMPORT_MAP",
"_COST_CALCULATOR_IMPORT_MAP",
@ -598,5 +613,6 @@ __all__ = [
"_DOTPROMPT_IMPORT_MAP",
"_TYPES_IMPORT_MAP",
"_LLM_CONFIGS_IMPORT_MAP",
"_LLM_PROVIDER_LOGIC_IMPORT_MAP",
]

62
litellm/google_genai/adapters/transformation.py
View file

@ -8,8 +8,10 @@ from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAssistantMessage,
ChatCompletionAssistantToolCall,
ChatCompletionImageObject,
ChatCompletionRequest,
ChatCompletionSystemMessage,
ChatCompletionTextObject,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolChoiceValues,
ChatCompletionToolMessage,
@ -385,13 +387,36 @@ class GoogleGenAIAdapter:
if role == "user":
# Handle user messages with potential function responses
combined_text = ""
content_parts: List[
Union[ChatCompletionTextObject, ChatCompletionImageObject]
] = []
tool_messages: List[ChatCompletionToolMessage] = []
for part in parts:
if isinstance(part, dict):
if "text" in part:
combined_text += part["text"]
content_parts.append(
cast(
ChatCompletionTextObject,
{"type": "text", "text": part["text"]},
)
)
elif "inline_data" in part:
# Handle Base64 image data
inline_data = part["inline_data"]
mime_type = inline_data.get("mime_type", "image/jpeg")
data = inline_data.get("data", "")
content_parts.append(
cast(
ChatCompletionImageObject,
{
"type": "image_url",
"image_url": {
"url": f"data:{mime_type};base64,{data}"
},
},
)
)
elif "functionResponse" in part:
# Transform function response to tool message
func_response = part["functionResponse"]
@ -402,13 +427,33 @@ class GoogleGenAIAdapter:
)
tool_messages.append(tool_message)
elif isinstance(part, str):
combined_text += part
content_parts.append(
cast(
ChatCompletionTextObject, {"type": "text", "text": part}
)
)
# Add user message if there's text content
if combined_text:
messages.append(
ChatCompletionUserMessage(role="user", content=combined_text)
)
# Add user message if there's content
if content_parts:
# If only one text part, use simple string format for backward compatibility
if (
len(content_parts) == 1
and isinstance(content_parts[0], dict)
and content_parts[0].get("type") == "text"
):
text_part = cast(ChatCompletionTextObject, content_parts[0])
messages.append(
ChatCompletionUserMessage(
role="user", content=text_part["text"]
)
)
else:
# Use multimodal format (array of content parts)
messages.append(
ChatCompletionUserMessage(
role="user", content=content_parts
)
)
# Add tool messages
messages.extend(tool_messages)
@ -468,7 +513,6 @@ class GoogleGenAIAdapter:
Dict in Google GenAI generate_content response format
"""
# Extract the main response content
choice = response.choices[0] if response.choices else None
if not choice:

76
litellm/integrations/generic_api/generic_api_callback.py
View file

@ -25,6 +25,7 @@ from litellm.llms.custom_httpx.http_handler import (
from litellm.types.utils import StandardLoggingPayload
API_EVENT_TYPES = Literal["llm_api_success", "llm_api_failure"]
LOG_FORMAT_TYPES = Literal["json_array", "ndjson", "single"]
def load_compatible_callbacks() -> Dict:
@ -101,6 +102,7 @@ class GenericAPILogger(CustomBatchLogger):
headers: Optional[dict] = None,
event_types: Optional[List[API_EVENT_TYPES]] = None,
callback_name: Optional[str] = None,
log_format: Optional[LOG_FORMAT_TYPES] = None,
**kwargs,
):
"""
@ -111,6 +113,7 @@ class GenericAPILogger(CustomBatchLogger):
headers: Optional[dict] = None,
event_types: Optional[List[API_EVENT_TYPES]] = None,
callback_name: Optional[str] = None - If provided, loads config from generic_api_compatible_callbacks.json
log_format: Optional[LOG_FORMAT_TYPES] = None - Format for log output: "json_array" (default), "ndjson", or "single"
"""
#########################################################
# Check if callback_name is provided and load config
@ -135,6 +138,9 @@ class GenericAPILogger(CustomBatchLogger):
if event_types is None and "event_types" in callback_config:
event_types = callback_config["event_types"]
if log_format is None and "log_format" in callback_config:
log_format = callback_config["log_format"]
else:
verbose_logger.warning(
f"callback_name '{callback_name}' not found in generic_api_compatible_callbacks.json"
@ -156,8 +162,16 @@ class GenericAPILogger(CustomBatchLogger):
self.endpoint: str = endpoint
self.event_types: Optional[List[API_EVENT_TYPES]] = event_types
self.callback_name: Optional[str] = callback_name
# Validate and store log_format
if log_format is not None and log_format not in ["json_array", "ndjson", "single"]:
raise ValueError(
f"Invalid log_format: {log_format}. Must be one of: 'json_array', 'ndjson', 'single'"
)
self.log_format: LOG_FORMAT_TYPES = log_format or "json_array"
verbose_logger.debug(
f"in init GenericAPILogger, callback_name: {self.callback_name}, endpoint {self.endpoint}, headers {self.headers}, event_types: {self.event_types}"
f"in init GenericAPILogger, callback_name: {self.callback_name}, endpoint {self.endpoint}, headers {self.headers}, event_types: {self.event_types}, log_format: {self.log_format}"
)
#########################################################
@ -289,25 +303,65 @@ class GenericAPILogger(CustomBatchLogger):
async def async_send_batch(self):
"""
Sends the batch of messages to Generic API Endpoint
Supports three formats:
- json_array: Sends all logs as a JSON array (default)
- ndjson: Sends logs as newline-delimited JSON
- single: Sends each log as individual HTTP request in parallel
"""
try:
if not self.log_queue:
return
verbose_logger.debug(
f"Generic API Logger - about to flush {len(self.log_queue)} events"
f"Generic API Logger - about to flush {len(self.log_queue)} events in '{self.log_format}' format"
)
# make POST request to Generic API Endpoint
response = await self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=safe_dumps(self.log_queue),
)
if self.log_format == "single":
# Send each log as individual HTTP request in parallel
tasks = []
for log_entry in self.log_queue:
task = self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=safe_dumps(log_entry),
)
tasks.append(task)
verbose_logger.debug(
f"Generic API Logger - sent batch to {self.endpoint}, status code {response.status_code}"
)
# Execute all requests in parallel
responses = await asyncio.gather(*tasks, return_exceptions=True)
# Log results
for idx, result in enumerate(responses):
if isinstance(result, Exception):
verbose_logger.exception(
f"Generic API Logger - Error sending log {idx}: {result}"
)
else:
# result is a Response object
verbose_logger.debug(
f"Generic API Logger - sent log {idx}, status: {result.status_code}" # type: ignore
)
else:
# Format the payload based on log_format
if self.log_format == "json_array":
data = safe_dumps(self.log_queue)
elif self.log_format == "ndjson":
data = "\n".join(safe_dumps(log) for log in self.log_queue)
else:
raise ValueError(f"Unknown log_format: {self.log_format}")
# Make POST request
response = await self.async_httpx_client.post(
url=self.endpoint,
headers=self.headers,
data=data,
)
verbose_logger.debug(
f"Generic API Logger - sent batch to {self.endpoint}, "
f"status: {response.status_code}, format: {self.log_format}"
)
except Exception as e:
verbose_logger.exception(

3
litellm/integrations/generic_api/generic_api_compatible_callbacks.json
View file

@ -22,6 +22,7 @@
"headers": {
"Content-Type": "application/json"
},
"environment_variables": ["SUMOLOGIC_WEBHOOK_URL"]
"environment_variables": ["SUMOLOGIC_WEBHOOK_URL"],
"log_format": "ndjson"
}
}

43
litellm/integrations/langfuse/langfuse.py
View file

@ -3,14 +3,27 @@
import os
import traceback
from datetime import datetime
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union, cast
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
List,
Optional,
Tuple,
Union,
cast,
)
from packaging.version import Version
import litellm
from litellm._logging import verbose_logger
from litellm.constants import MAX_LANGFUSE_INITIALIZED_CLIENTS
from litellm.litellm_core_utils.core_helpers import safe_deep_copy
from litellm.litellm_core_utils.core_helpers import (
safe_deep_copy,
reconstruct_model_name,
)
from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
from litellm.llms.custom_httpx.http_handler import _get_httpx_client
from litellm.secret_managers.main import str_to_bool
@ -437,12 +450,17 @@ class LangFuseLogger:
)
)
custom_llm_provider = cast(Optional[str], kwargs.get("custom_llm_provider"))
model_name = reconstruct_model_name(
kwargs.get("model", ""), custom_llm_provider, metadata
)
trace.generation(
CreateGeneration(
name=metadata.get("generation_name", "litellm-completion"),
startTime=start_time,
endTime=end_time,
model=kwargs["model"],
model=model_name,
modelParameters=optional_params,
prompt=input,
completion=output,
@ -543,7 +561,9 @@ class LangFuseLogger:
# as we want to fall back to litellm_call_id instead for better traceability.
# Note: Users can still explicitly set a UUID trace_id via metadata["trace_id"] (highest priority)
if trace_id is None and standard_logging_object is not None:
standard_trace_id = cast(Optional[str], standard_logging_object.get("trace_id"))
standard_trace_id = cast(
Optional[str], standard_logging_object.get("trace_id")
)
# Only use standard_logging_object.trace_id if it's not a UUID
# UUIDs are 36 characters with hyphens in format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
# We check for this specific pattern to avoid rejecting valid trace_ids that happen to have hyphens
@ -575,7 +595,9 @@ class LangFuseLogger:
mask_output = clean_metadata.pop("mask_output", False)
# Look for masking function in the dedicated location first (set by scrub_sensitive_keys_in_metadata)
# Fall back to metadata for backwards compatibility
masking_function = litellm_params.get("_langfuse_masking_function") or clean_metadata.pop("langfuse_masking_function", None)
masking_function = litellm_params.get(
"_langfuse_masking_function"
) or clean_metadata.pop("langfuse_masking_function", None)
# Apply custom masking function if provided
if masking_function is not None and callable(masking_function):
@ -776,12 +798,17 @@ class LangFuseLogger:
if system_fingerprint is not None:
optional_params["system_fingerprint"] = system_fingerprint
custom_llm_provider = cast(Optional[str], kwargs.get("custom_llm_provider"))
model_name = reconstruct_model_name(
kwargs.get("model", ""), custom_llm_provider, metadata
)
generation_params = {
"name": generation_name,
"id": clean_metadata.pop("generation_id", generation_id),
"start_time": start_time,
"end_time": end_time,
"model": kwargs["model"],
"model": model_name,
"model_parameters": optional_params,
"input": input if not mask_input else "redacted-by-litellm",
"output": output if not mask_output else "redacted-by-litellm",
@ -918,7 +945,9 @@ class LangFuseLogger:
return Version(self.langfuse_sdk_version) >= Version("2.7.3")
@staticmethod
def _apply_masking_function(data: Any, masking_function: Callable[[Any], Any]) -> Any:
def _apply_masking_function(
data: Any, masking_function: Callable[[Any], Any]
) -> Any:
"""
Apply a masking function to data, handling different data types.

181
litellm/integrations/opentelemetry.py
View file

@ -48,6 +48,7 @@ else:
LITELLM_TRACER_NAME = os.getenv("OTEL_TRACER_NAME", "litellm")
LITELLM_METER_NAME = os.getenv("LITELLM_METER_NAME", "litellm")
LITELLM_LOGGER_NAME = os.getenv("LITELLM_LOGGER_NAME", "litellm")
LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
# Remove the hardcoded LITELLM_RESOURCE dictionary - we'll create it properly later
RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
LITELLM_REQUEST_SPAN_NAME = "litellm_request"
@ -233,14 +234,16 @@ class OpenTelemetry(CustomLogger):
trace.set_tracer_provider(tracer_provider)
else:
# Tracer provider explicitly provided (e.g., for testing)
# Do NOT call set_tracer_provider - the caller is responsible for managing global state
# If they want it to be global, they've already set it before passing it to us
verbose_logger.debug(
"OpenTelemetry: Using provided TracerProvider: %s",
type(tracer_provider).__name__,
)
trace.set_tracer_provider(tracer_provider)
# grab our tracer
self.tracer = trace.get_tracer(LITELLM_TRACER_NAME)
# Grab our tracer from the TracerProvider (not from global context)
# This ensures we use the provided TracerProvider (e.g., for testing)
self.tracer = tracer_provider.get_tracer(LITELLM_TRACER_NAME)
self.span_kind = SpanKind
def _init_metrics(self, meter_provider):
@ -527,6 +530,7 @@ class OpenTelemetry(CustomLogger):
# 3. Guardrail span
self._create_guardrail_span(kwargs=kwargs, context=ctx)
return response
#########################################################
@ -557,9 +561,9 @@ class OpenTelemetry(CustomLogger):
def _get_dynamic_otel_headers_from_kwargs(self, kwargs) -> Optional[dict]:
"""Extract dynamic headers from kwargs if available."""
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params")
)
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params")
if not standard_callback_dynamic_params:
return None
@ -607,18 +611,35 @@ class OpenTelemetry(CustomLogger):
)
ctx, parent_span = self._get_span_context(kwargs)
if get_secret_bool("USE_OTEL_LITELLM_REQUEST_SPAN"):
primary_span_parent = None
else:
primary_span_parent = parent_span
# 1. Primary span
span = self._start_primary_span(
kwargs, response_obj, start_time, end_time, ctx, primary_span_parent
# Decide whether to create a primary span
# Always create if no parent span exists (backward compatibility)
# OR if USE_OTEL_LITELLM_REQUEST_SPAN is explicitly enabled
should_create_primary_span = parent_span is None or get_secret_bool(
"USE_OTEL_LITELLM_REQUEST_SPAN"
)
# 2. Raw‐request sub-span (if enabled)
self._maybe_log_raw_request(kwargs, response_obj, start_time, end_time, span)
if should_create_primary_span:
# Create a new litellm_request span
span = self._start_primary_span(
kwargs, response_obj, start_time, end_time, ctx
)
# Raw-request sub-span (if enabled) - child of litellm_request span
self._maybe_log_raw_request(
kwargs, response_obj, start_time, end_time, span
)
else:
# Do not create primary span (keep hierarchy shallow when parent exists)
from opentelemetry.trace import Status, StatusCode
span = None
# Only set attributes if the span is still recording (not closed)
# Note: parent_span is guaranteed to be not None here
parent_span.set_status(Status(StatusCode.OK))
self.set_attributes(parent_span, kwargs, response_obj)
# Raw-request as direct child of parent_span
self._maybe_log_raw_request(
kwargs, response_obj, start_time, end_time, parent_span
)
# 3. Guardrail span
self._create_guardrail_span(kwargs=kwargs, context=ctx)
@ -628,12 +649,18 @@ class OpenTelemetry(CustomLogger):
# 5. Semantic logs.
if self.config.enable_events:
self._emit_semantic_logs(kwargs, response_obj, span)
log_span = span if span is not None else parent_span
if log_span is not None:
self._emit_semantic_logs(kwargs, response_obj, log_span)
# 6. End parent span (only if it wasn't reused as the primary span)
# If parent_span was reused as the primary span, it was already ended in _start_primary_span
if parent_span is not None and parent_span is not span:
parent_span.end(end_time=self._to_ns(datetime.now()))
# 6. Do NOT end parent span - it should be managed by its creator
# External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
# However, proxy-created spans should be closed here
if (
parent_span is not None
and parent_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
):
parent_span.end(end_time=self._to_ns(end_time))
def _start_primary_span(
self,
@ -642,16 +669,19 @@ class OpenTelemetry(CustomLogger):
start_time,
end_time,
context,
parent_span: Optional[Span] = None,
):
from opentelemetry.trace import Status, StatusCode
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
span = parent_span or otel_tracer.start_span(
# Always create a new span
# The parent relationship is preserved through the context parameter
span = otel_tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=context,
)
span.set_status(Status(StatusCode.OK))
self.set_attributes(span, kwargs, response_obj)
span.end(end_time=self._to_ns(end_time))
@ -764,10 +794,10 @@ class OpenTelemetry(CustomLogger):
return float(val)
# isinstance(val, str) - parse datetime string (with or without microseconds)
try:
return datetime.strptime(val, '%Y-%m-%d %H:%M:%S.%f').timestamp()
return datetime.strptime(val, "%Y-%m-%d %H:%M:%S.%f").timestamp()
except ValueError:
try:
return datetime.strptime(val, '%Y-%m-%d %H:%M:%S').timestamp()
return datetime.strptime(val, "%Y-%m-%d %H:%M:%S").timestamp()
except ValueError:
return None
@ -775,23 +805,23 @@ class OpenTelemetry(CustomLogger):
"""Record Time to First Token (TTFT) metric for streaming requests."""
optional_params = kwargs.get("optional_params", {})
is_streaming = optional_params.get("stream", False)
if not (self._time_to_first_token_histogram and is_streaming):
return
# Use api_call_start_time for precision (matches Prometheus implementation)
# This excludes LiteLLM overhead and measures pure LLM API latency
api_call_start_time = kwargs.get("api_call_start_time", None)
completion_start_time = kwargs.get("completion_start_time", None)
if api_call_start_time is not None and completion_start_time is not None:
# Convert to timestamps if needed (handles datetime, float, and string)
api_call_start_ts = self._to_timestamp(api_call_start_time)
completion_start_ts = self._to_timestamp(completion_start_time)
if api_call_start_ts is None or completion_start_ts is None:
return # Skip recording if conversion failed
time_to_first_token_seconds = completion_start_ts - api_call_start_ts
self._time_to_first_token_histogram.record(
time_to_first_token_seconds, attributes=common_attrs
@ -806,38 +836,40 @@ class OpenTelemetry(CustomLogger):
common_attrs: dict,
):
"""Record Time Per Output Token (TPOT) metric.
Calculated as: generation_time / completion_tokens
- For streaming: uses end_time - completion_start_time (time to generate all tokens after first)
- For non-streaming: uses end_time - api_call_start_time (total generation time)
"""
if not self._time_per_output_token_histogram:
return
# Get completion tokens from response_obj
completion_tokens = None
if response_obj and (usage := response_obj.get("usage")):
completion_tokens = usage.get("completion_tokens")
if completion_tokens is None or completion_tokens <= 0:
return
# Calculate generation time
completion_start_time = kwargs.get("completion_start_time", None)
api_call_start_time = kwargs.get("api_call_start_time", None)
# Convert end_time to timestamp (handles datetime, float, and string)
end_time_ts = self._to_timestamp(end_time)
if end_time_ts is None:
# Fallback to duration_s if conversion failed
generation_time_seconds = duration_s
if generation_time_seconds > 0:
time_per_output_token_seconds = generation_time_seconds / completion_tokens
time_per_output_token_seconds = (
generation_time_seconds / completion_tokens
)
self._time_per_output_token_histogram.record(
time_per_output_token_seconds, attributes=common_attrs
)
return
if completion_start_time is not None:
# Streaming: use completion_start_time (when first token arrived)
# This measures time to generate all tokens after the first one
@ -858,7 +890,7 @@ class OpenTelemetry(CustomLogger):
else:
# Fallback: use duration_s (already calculated as (end_time - start_time).total_seconds())
generation_time_seconds = duration_s
if generation_time_seconds > 0:
time_per_output_token_seconds = generation_time_seconds / completion_tokens
self._time_per_output_token_histogram.record(
@ -872,37 +904,37 @@ class OpenTelemetry(CustomLogger):
common_attrs: dict,
):
"""Record Total Generation Time (response duration) metric.
Measures pure LLM API generation time: end_time - api_call_start_time
This excludes LiteLLM overhead and measures only the LLM provider's response time.
Works for both streaming and non-streaming requests.
Mirrors Prometheus's litellm_llm_api_latency_metric.
Uses kwargs.get("end_time") with fallback to parameter for consistency with Prometheus.
"""
if not self._response_duration_histogram:
return
api_call_start_time = kwargs.get("api_call_start_time", None)
if api_call_start_time is None:
return
# Use end_time from kwargs if available (matches Prometheus), otherwise use parameter
# For streaming: end_time is when the stream completes (final chunk received)
# For non-streaming: end_time is when the response is received
_end_time = kwargs.get("end_time") or end_time
if _end_time is None:
_end_time = datetime.now()
# Convert to timestamps if needed (handles datetime, float, and string)
api_call_start_ts = self._to_timestamp(api_call_start_time)
end_time_ts = self._to_timestamp(_end_time)
if api_call_start_ts is None or end_time_ts is None:
return # Skip recording if conversion failed
response_duration_seconds = end_time_ts - api_call_start_ts
if response_duration_seconds > 0:
self._response_duration_histogram.record(
response_duration_seconds, attributes=common_attrs
@ -1065,26 +1097,49 @@ class OpenTelemetry(CustomLogger):
)
_parent_context, parent_otel_span = self._get_span_context(kwargs)
# Span 1: Requst sent to litellm SDK
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
span = otel_tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=_parent_context,
# Decide whether to create a primary span
# Always create if no parent span exists (backward compatibility)
# OR if USE_OTEL_LITELLM_REQUEST_SPAN is explicitly enabled
should_create_primary_span = parent_otel_span is None or get_secret_bool(
"USE_OTEL_LITELLM_REQUEST_SPAN"
)
span.set_status(Status(StatusCode.ERROR))
self.set_attributes(span, kwargs, response_obj)
# Record exception information using OTEL standard method
self._record_exception_on_span(span=span, kwargs=kwargs)
if should_create_primary_span:
# Span 1: Request sent to litellm SDK
otel_tracer: Tracer = self.get_tracer_to_use_for_request(kwargs)
span = otel_tracer.start_span(
name=self._get_span_name(kwargs),
start_time=self._to_ns(start_time),
context=_parent_context,
)
span.set_status(Status(StatusCode.ERROR))
self.set_attributes(span, kwargs, response_obj)
span.end(end_time=self._to_ns(end_time))
# Record exception information using OTEL standard method
self._record_exception_on_span(span=span, kwargs=kwargs)
span.end(end_time=self._to_ns(end_time))
else:
# When parent span exists and USE_OTEL_LITELLM_REQUEST_SPAN=false,
# record error on parent span (keeps hierarchy shallow)
# Only set attributes if the span is still recording (not closed)
# Note: parent_otel_span is guaranteed to be not None here
if parent_otel_span.is_recording():
parent_otel_span.set_status(Status(StatusCode.ERROR))
self.set_attributes(parent_otel_span, kwargs, response_obj)
self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
# Create span for guardrail information
self._create_guardrail_span(kwargs=kwargs, context=_parent_context)
if parent_otel_span is not None:
parent_otel_span.end(end_time=self._to_ns(datetime.now()))
# Do NOT end parent span - it should be managed by its creator
# External spans (from Langfuse, user code, HTTP headers, global context) must not be closed by LiteLLM
# However, proxy-created spans should be closed here
if (
parent_otel_span is not None
and parent_otel_span.name == LITELLM_PROXY_REQUEST_SPAN_NAME
):
parent_otel_span.end(end_time=self._to_ns(end_time))
def _record_exception_on_span(self, span: Span, kwargs: dict):
"""
@ -1263,7 +1318,9 @@ class OpenTelemetry(CustomLogger):
)
return
elif self.callback_name == "weave_otel":
from litellm.integrations.weave.weave_otel import set_weave_otel_attributes
from litellm.integrations.weave.weave_otel import (
set_weave_otel_attributes,
)
set_weave_otel_attributes(span, kwargs, response_obj)
return
@ -1994,9 +2051,9 @@ class OpenTelemetry(CustomLogger):
"""
Create a span for the received proxy server request.
"""
return self.tracer.start_span(
name="Received Proxy Server Request",
name=LITELLM_PROXY_REQUEST_SPAN_NAME,
start_time=self._to_ns(start_time),
context=self.get_traceparent_from_header(headers=headers),
kind=self.span_kind.SERVER,

4
litellm/integrations/prometheus.py
View file

@ -214,7 +214,7 @@ class PrometheusLogger(CustomLogger):
# Remaining Rate Limit for model
self.litellm_remaining_requests_metric = self._gauge_factory(
"litellm_remaining_requests",
"litellm_remaining_requests_metric",
"LLM Deployment Analytics - remaining requests for model, returned from LLM API Provider",
labelnames=self.get_labels_for_metric(
"litellm_remaining_requests_metric"
@ -222,7 +222,7 @@ class PrometheusLogger(CustomLogger):
)
self.litellm_remaining_tokens_metric = self._gauge_factory(
"litellm_remaining_tokens",
"litellm_remaining_tokens_metric",
"remaining tokens for model, returned from LLM API Provider",
labelnames=self.get_labels_for_metric(
"litellm_remaining_tokens_metric"

72
litellm/litellm_core_utils/core_helpers.py
View file

@ -38,18 +38,18 @@ def safe_divide_seconds(
def safe_divide(
numerator: Union[int, float],
denominator: Union[int, float],
default: Union[int, float] = 0
numerator: Union[int, float],
denominator: Union[int, float],
default: Union[int, float] = 0,
) -> Union[int, float]:
"""
Safely divide two numbers, returning a default value if denominator is zero.
Args:
numerator: The number to divide
denominator: The number to divide by
default: Value to return if denominator is zero (defaults to 0)
Returns:
The result of numerator/denominator, or default if denominator is zero
"""
@ -153,7 +153,8 @@ def get_metadata_variable_name_from_kwargs(
- LiteLLM is now moving to using `litellm_metadata` for our metadata
"""
return "litellm_metadata" if "litellm_metadata" in kwargs else "metadata"
def get_litellm_metadata_from_kwargs(kwargs: dict):
"""
Helper to get litellm metadata from all litellm request kwargs
@ -176,6 +177,25 @@ def get_litellm_metadata_from_kwargs(kwargs: dict):
return {}
def reconstruct_model_name(
model_name: str,
custom_llm_provider: Optional[str],
metadata: dict,
) -> str:
"""Reconstruct full model name with provider prefix for logging."""
# Check if deployment model name from router metadata is available (has original prefix)
deployment_model_name = metadata.get("deployment")
if deployment_model_name and "/" in deployment_model_name:
# Use the deployment model name which preserves the original provider prefix
return deployment_model_name
elif custom_llm_provider and model_name and "/" not in model_name:
# Only add prefix for Bedrock (not for direct Anthropic API)
# This ensures Bedrock models get the prefix while direct Anthropic models don't
if custom_llm_provider == "bedrock":
return f"{custom_llm_provider}/{model_name}"
return model_name
# Helper functions used for OTEL logging
def _get_parent_otel_span_from_kwargs(
kwargs: Optional[dict] = None,
@ -246,8 +266,8 @@ def safe_deep_copy(data):
Safe Deep Copy
The LiteLLM request may contain objects that cannot be pickled/deep-copied
(e.g., tracing spans, locks, clients).
(e.g., tracing spans, locks, clients).
This helper deep-copies each top-level key independently; on failure keeps
original ref
"""
@ -306,23 +326,23 @@ def safe_deep_copy(data):
def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
"""
Recursively filter out Exception objects and callable objects from dicts/lists.
This is a defensive utility to prevent deepcopy failures when exception objects
are accidentally stored in parameter dictionaries (e.g., optional_params).
Also filters callable objects (functions) to prevent JSON serialization errors.
Exceptions and callables should not be stored in params - this function removes them.
Args:
data: The data structure to filter (dict, list, or any other type)
max_depth: Maximum recursion depth to prevent infinite loops
Returns:
Filtered data structure with Exception and callable objects removed, or None if the
entire input was an Exception or callable
"""
if max_depth <= 0:
return data
# Skip exception objects
if isinstance(data, Exception):
return None
@ -333,7 +353,7 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
obj_type_name = type(data).__name__
if obj_type_name in ["Logging", "LiteLLMLoggingObj"]:
return None
if isinstance(data, dict):
result: dict[str, Any] = {}
for k, v in data.items():
@ -352,7 +372,9 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
result_list: list[Any] = []
for item in data:
# Skip exception and callable items
if isinstance(item, Exception) or (callable(item) and not isinstance(item, type)):
if isinstance(item, Exception) or (
callable(item) and not isinstance(item, type)
):
continue
try:
filtered = filter_exceptions_from_params(item, max_depth - 1)
@ -366,37 +388,35 @@ def filter_exceptions_from_params(data: Any, max_depth: int = 20) -> Any:
return data
def filter_internal_params(data: dict, additional_internal_params: Optional[set] = None) -> dict:
def filter_internal_params(
data: dict, additional_internal_params: Optional[set] = None
) -> dict:
"""
Filter out LiteLLM internal parameters that shouldn't be sent to provider APIs.
This removes internal/MCP-related parameters that are used by LiteLLM internally
but should not be included in API requests to providers.
Args:
data: Dictionary of parameters to filter
additional_internal_params: Optional set of additional internal parameter names to filter
Returns:
Filtered dictionary with internal parameters removed
"""
if not isinstance(data, dict):
return data
# Known internal parameters that should never be sent to provider APIs
internal_params = {
"skip_mcp_handler",
"mcp_handler_context",
"_skip_mcp_handler",
}
# Add any additional internal params if provided
if additional_internal_params:
internal_params.update(additional_internal_params)
# Filter out internal parameters
return {
k: v
for k, v in data.items()
if k not in internal_params
}
return {k: v for k, v in data.items() if k not in internal_params}

11
litellm/litellm_core_utils/dot_notation_indexing.py
View file

@ -9,6 +9,7 @@ Custom implementation with zero external dependencies.
Supported syntax:
- "field" - top-level field
- "parent.child" - nested field
- "parent\\.with\\.dots.child" - keys containing dots (escape with backslash)
- "array[*]" - all array elements (wildcard)
- "array[0]" - specific array element (index)
- "array[*].field" - field in all array elements
@ -47,6 +48,9 @@ def get_nested_value(
'value'
>>> get_nested_value(data, "a.b.d", "default")
'default'
>>> data = {"kubernetes.io": {"namespace": "default"}}
>>> get_nested_value(data, "kubernetes\\.io.namespace")
'default'
"""
if not key_path:
return default
@ -58,8 +62,11 @@ def get_nested_value(
else key_path
)
# Split the key path into parts
parts = key_path.split(".")
# Split the key path into parts, respecting escaped dots (\.)
# Use a temporary placeholder, split on unescaped dots, then restore
placeholder = "\x00"
parts = key_path.replace("\\.", placeholder).split(".")
parts = [p.replace(placeholder, ".") for p in parts]
# Traverse through the dictionary
current: Any = data

4
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -229,10 +229,10 @@ def get_llm_provider( # noqa: PLR0915
elif endpoint == "https://api.ai21.com/studio/v1":
custom_llm_provider = "ai21_chat"
dynamic_api_key = get_secret_str("AI21_API_KEY")
elif endpoint == "https://codestral.mistral.ai/v1":
elif endpoint == "codestral.mistral.ai/v1/chat/completions":
custom_llm_provider = "codestral"
dynamic_api_key = get_secret_str("CODESTRAL_API_KEY")
elif endpoint == "https://codestral.mistral.ai/v1":
elif endpoint == "codestral.mistral.ai/v1/fim/completions":
custom_llm_provider = "text-completion-codestral"
dynamic_api_key = get_secret_str("CODESTRAL_API_KEY")
elif endpoint == "app.empower.dev/api/v1":

351
litellm/litellm_core_utils/litellm_logging.py
View file

@ -59,6 +59,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.core_helpers import reconstruct_model_name
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
@ -332,9 +333,9 @@ class Logging(LiteLLMLoggingBaseClass):
self.litellm_trace_id: str = litellm_trace_id or str(uuid.uuid4())
self.function_id = function_id
self.streaming_chunks: List[Any] = [] # for generating complete stream response
self.sync_streaming_chunks: List[Any] = (
[]
) # for generating complete stream response
self.sync_streaming_chunks: List[
Any
] = [] # for generating complete stream response
self.log_raw_request_response = log_raw_request_response
# Initialize dynamic callbacks
@ -719,9 +720,9 @@ class Logging(LiteLLMLoggingBaseClass):
prompt_spec=prompt_spec,
dynamic_callback_params=dynamic_callback_params,
):
self.model_call_details["prompt_integration"] = (
logger.__class__.__name__
)
self.model_call_details[
"prompt_integration"
] = logger.__class__.__name__
return logger
except Exception:
# If check fails, continue to next logger
@ -789,9 +790,9 @@ class Logging(LiteLLMLoggingBaseClass):
if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
non_default_params
):
self.model_call_details["prompt_integration"] = (
anthropic_cache_control_logger.__class__.__name__
)
self.model_call_details[
"prompt_integration"
] = anthropic_cache_control_logger.__class__.__name__
return anthropic_cache_control_logger
#########################################################
@ -803,9 +804,9 @@ class Logging(LiteLLMLoggingBaseClass):
internal_usage_cache=None,
llm_router=None,
)
self.model_call_details["prompt_integration"] = (
vector_store_custom_logger.__class__.__name__
)
self.model_call_details[
"prompt_integration"
] = vector_store_custom_logger.__class__.__name__
# Add to global callbacks so post-call hooks are invoked
if (
vector_store_custom_logger
@ -865,9 +866,9 @@ class Logging(LiteLLMLoggingBaseClass):
model
): # if model name was changes pre-call, overwrite the initial model call name with the new one
self.model_call_details["model"] = model
self.model_call_details["litellm_params"]["api_base"] = (
self._get_masked_api_base(additional_args.get("api_base", ""))
)
self.model_call_details["litellm_params"][
"api_base"
] = self._get_masked_api_base(additional_args.get("api_base", ""))
def pre_call(self, input, api_key, model=None, additional_args={}): # noqa: PLR0915
# Log the exact input to the LLM API
@ -896,10 +897,10 @@ class Logging(LiteLLMLoggingBaseClass):
try:
# [Non-blocking Extra Debug Information in metadata]
if turn_off_message_logging is True:
_metadata["raw_request"] = (
"redacted by litellm. \
_metadata[
"raw_request"
] = "redacted by litellm. \
'litellm.turn_off_message_logging=True'"
)
else:
curl_command = self._get_request_curl_command(
api_base=additional_args.get("api_base", ""),
@ -910,34 +911,34 @@ class Logging(LiteLLMLoggingBaseClass):
_metadata["raw_request"] = str(curl_command)
# split up, so it's easier to parse in the UI
self.model_call_details["raw_request_typed_dict"] = (
RawRequestTypedDict(
raw_request_api_base=str(
additional_args.get("api_base") or ""
),
raw_request_body=self._get_raw_request_body(
additional_args.get("complete_input_dict", {})
),
# NOTE: setting ignore_sensitive_headers to True will cause
# the Authorization header to be leaked when calls to the health
# endpoint are made and fail.
raw_request_headers=self._get_masked_headers(
additional_args.get("headers", {}) or {},
),
error=None,
)
self.model_call_details[
"raw_request_typed_dict"
] = RawRequestTypedDict(
raw_request_api_base=str(
additional_args.get("api_base") or ""
),
raw_request_body=self._get_raw_request_body(
additional_args.get("complete_input_dict", {})
),
# NOTE: setting ignore_sensitive_headers to True will cause
# the Authorization header to be leaked when calls to the health
# endpoint are made and fail.
raw_request_headers=self._get_masked_headers(
additional_args.get("headers", {}) or {},
),
error=None,
)
except Exception as e:
self.model_call_details["raw_request_typed_dict"] = (
RawRequestTypedDict(
error=str(e),
)
self.model_call_details[
"raw_request_typed_dict"
] = RawRequestTypedDict(
error=str(e),
)
_metadata["raw_request"] = (
"Unable to Log \
_metadata[
"raw_request"
] = "Unable to Log \
raw request: {}".format(
str(e)
)
str(e)
)
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
@ -1238,13 +1239,13 @@ class Logging(LiteLLMLoggingBaseClass):
for callback in callbacks:
try:
if isinstance(callback, CustomLogger):
response: Optional[MCPPostCallResponseObject] = (
await callback.async_post_mcp_tool_call_hook(
kwargs=kwargs,
response_obj=post_mcp_tool_call_response_obj,
start_time=start_time,
end_time=end_time,
)
response: Optional[
MCPPostCallResponseObject
] = await callback.async_post_mcp_tool_call_hook(
kwargs=kwargs,
response_obj=post_mcp_tool_call_response_obj,
start_time=start_time,
end_time=end_time,
)
######################################################################
# if any of the callbacks modify the response, use the modified response
@ -1423,9 +1424,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
f"response_cost_failure_debug_information: {debug_info}"
)
self.model_call_details["response_cost_failure_debug_information"] = (
debug_info
)
self.model_call_details[
"response_cost_failure_debug_information"
] = debug_info
return None
try:
@ -1451,9 +1452,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
f"response_cost_failure_debug_information: {debug_info}"
)
self.model_call_details["response_cost_failure_debug_information"] = (
debug_info
)
self.model_call_details[
"response_cost_failure_debug_information"
] = debug_info
return None
@ -1603,16 +1604,16 @@ class Logging(LiteLLMLoggingBaseClass):
result=logging_result
)
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
def _transform_usage_objects(self, result):
@ -1667,9 +1668,9 @@ class Logging(LiteLLMLoggingBaseClass):
end_time = datetime.datetime.now()
if self.completion_start_time is None:
self.completion_start_time = end_time
self.model_call_details["completion_start_time"] = (
self.completion_start_time
)
self.model_call_details[
"completion_start_time"
] = self.completion_start_time
self.model_call_details["log_event_type"] = "successful_api_call"
self.model_call_details["end_time"] = end_time
@ -1706,21 +1707,21 @@ class Logging(LiteLLMLoggingBaseClass):
end_time=end_time,
)
elif isinstance(result, dict) or isinstance(result, list):
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
elif standard_logging_object is not None:
self.model_call_details["standard_logging_object"] = (
standard_logging_object
)
self.model_call_details[
"standard_logging_object"
] = standard_logging_object
else:
self.model_call_details["response_cost"] = None
@ -1870,23 +1871,23 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
"Logging Details LiteLLM-Success Call streaming complete"
)
self.model_call_details["complete_streaming_response"] = (
complete_streaming_response
)
self.model_call_details["response_cost"] = (
self._response_cost_calculator(result=complete_streaming_response)
)
self.model_call_details[
"complete_streaming_response"
] = complete_streaming_response
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(result=complete_streaming_response)
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
callbacks = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_success_callbacks,
@ -2214,10 +2215,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
else:
if self.stream and complete_streaming_response:
self.model_call_details["complete_response"] = (
self.model_call_details.get(
"complete_streaming_response", {}
)
self.model_call_details[
"complete_response"
] = self.model_call_details.get(
"complete_streaming_response", {}
)
result = self.model_call_details["complete_response"]
openMeterLogger.log_success_event(
@ -2256,10 +2257,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
else:
if self.stream and complete_streaming_response:
self.model_call_details["complete_response"] = (
self.model_call_details.get(
"complete_streaming_response", {}
)
self.model_call_details[
"complete_response"
] = self.model_call_details.get(
"complete_streaming_response", {}
)
result = self.model_call_details["complete_response"]
@ -2402,9 +2403,9 @@ class Logging(LiteLLMLoggingBaseClass):
if complete_streaming_response is not None:
print_verbose("Async success callbacks: Got a complete streaming response")
self.model_call_details["async_complete_streaming_response"] = (
complete_streaming_response
)
self.model_call_details[
"async_complete_streaming_response"
] = complete_streaming_response
try:
if self.model_call_details.get("cache_hit", False) is True:
@ -2415,10 +2416,10 @@ class Logging(LiteLLMLoggingBaseClass):
model_call_details=self.model_call_details
)
# base_model defaults to None if not set on model_info
self.model_call_details["response_cost"] = (
self._response_cost_calculator(
result=complete_streaming_response
)
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(
result=complete_streaming_response
)
verbose_logger.debug(
@ -2431,16 +2432,16 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = None
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
callbacks = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_async_success_callbacks,
@ -2676,18 +2677,18 @@ class Logging(LiteLLMLoggingBaseClass):
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj={},
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="failure",
error_str=str(exception),
original_exception=exception,
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj={},
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="failure",
error_str=str(exception),
original_exception=exception,
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
return start_time, end_time
@ -3301,7 +3302,9 @@ class Logging(LiteLLMLoggingBaseClass):
# Deep copy result and add usage
result_copy = result.model_copy(deep=True)
result_copy.usage = usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)
result_copy.usage = (
usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)
)
return result_copy
@ -3629,9 +3632,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
endpoint=arize_config.endpoint,
)
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = f"space_id={arize_config.space_key or arize_config.space_id},api_key={arize_config.api_key}"
for callback in _in_memory_loggers:
if (
isinstance(callback, ArizeLogger)
@ -3642,7 +3645,6 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
_in_memory_loggers.append(_arize_otel_logger)
return _arize_otel_logger # type: ignore
elif logging_integration == "arize_phoenix":
from litellm.integrations.opentelemetry import (
OpenTelemetry,
OpenTelemetryConfig,
@ -3658,13 +3660,13 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
# Add openinference.project.name attribute
if existing_attrs:
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
)
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"{existing_attrs},openinference.project.name={arize_phoenix_config.project_name}"
else:
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"openinference.project.name={arize_phoenix_config.project_name}"
)
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"openinference.project.name={arize_phoenix_config.project_name}"
# Set Phoenix project name from environment variable
phoenix_project_name = os.environ.get("PHOENIX_PROJECT_NAME", None)
@ -3672,19 +3674,19 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
existing_attrs = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "")
# Add openinference.project.name attribute
if existing_attrs:
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"{existing_attrs},openinference.project.name={phoenix_project_name}"
)
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"{existing_attrs},openinference.project.name={phoenix_project_name}"
else:
os.environ["OTEL_RESOURCE_ATTRIBUTES"] = (
f"openinference.project.name={phoenix_project_name}"
)
os.environ[
"OTEL_RESOURCE_ATTRIBUTES"
] = f"openinference.project.name={phoenix_project_name}"
# auth can be disabled on local deployments of arize phoenix
if arize_phoenix_config.otlp_auth_headers is not None:
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
arize_phoenix_config.otlp_auth_headers
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = arize_phoenix_config.otlp_auth_headers
for callback in _in_memory_loggers:
if (
@ -3816,9 +3818,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
exporter="otlp_http",
endpoint="https://langtrace.ai/api/trace",
)
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
f"api_key={os.getenv('LANGTRACE_API_KEY')}"
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
for callback in _in_memory_loggers:
if (
isinstance(callback, OpenTelemetry)
@ -4589,10 +4591,10 @@ class StandardLoggingPayloadSetup:
for key in StandardLoggingHiddenParams.__annotations__.keys():
if key in hidden_params:
if key == "additional_headers":
clean_hidden_params["additional_headers"] = (
StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
)
clean_hidden_params[
"additional_headers"
] = StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
)
else:
clean_hidden_params[key] = hidden_params[key] # type: ignore
@ -4898,25 +4900,6 @@ def _extract_response_obj_and_hidden_params(
return response_obj, hidden_params
def _reconstruct_model_name(
model_name: str,
custom_llm_provider: Optional[str],
metadata: dict,
) -> str:
"""Reconstruct full model name with provider prefix for logging."""
# Check if deployment model name from router metadata is available (has original prefix)
deployment_model_name = metadata.get("deployment")
if deployment_model_name and "/" in deployment_model_name:
# Use the deployment model name which preserves the original provider prefix
return deployment_model_name
elif custom_llm_provider and model_name and "/" not in model_name:
# Only add prefix for Bedrock (not for direct Anthropic API)
# This ensures Bedrock models get the prefix while direct Anthropic models don't
if custom_llm_provider == "bedrock":
return f"{custom_llm_provider}/{model_name}"
return model_name
def get_standard_logging_object_payload(
kwargs: Optional[dict],
init_response_obj: Union[Any, BaseModel, dict],
@ -5049,7 +5032,7 @@ def get_standard_logging_object_payload(
# This ensures Bedrock models like "us.anthropic.claude-3-5-sonnet-20240620-v1:0"
# are logged as "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0"
custom_llm_provider = cast(Optional[str], kwargs.get("custom_llm_provider"))
model_name = _reconstruct_model_name(
model_name = reconstruct_model_name(
kwargs.get("model", "") or "", custom_llm_provider, metadata
)
@ -5205,9 +5188,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
):
for k, v in metadata["user_api_key_metadata"].items():
if k == "logging": # prevent logging user logging keys
cleaned_user_api_key_metadata[k] = (
"scrubbed_by_litellm_for_sensitive_keys"
)
cleaned_user_api_key_metadata[
k
] = "scrubbed_by_litellm_for_sensitive_keys"
else:
cleaned_user_api_key_metadata[k] = v

85
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -161,6 +161,15 @@ def _get_token_base_cost(
prompt_base_cost = cast(float, _get_cost_per_unit(model_info, input_cost_key))
completion_base_cost = cast(float, _get_cost_per_unit(model_info, output_cost_key))
# For image generation models that don't have output_cost_per_token,
# use output_cost_per_image_token as the base cost (all output tokens are image tokens)
if completion_base_cost == 0.0 or completion_base_cost is None:
output_image_cost = _get_cost_per_unit(
model_info, "output_cost_per_image_token", None
)
if output_image_cost is not None:
completion_base_cost = cast(float, output_image_cost)
cache_creation_cost = cast(
float, _get_cost_per_unit(model_info, cache_creation_cost_key)
)
@ -342,6 +351,7 @@ class PromptTokensDetailsResult(TypedDict):
cache_creation_token_details: Optional[CacheCreationTokenDetails]
text_tokens: int
audio_tokens: int
image_tokens: int
character_count: int
image_count: int
video_length_seconds: int
@ -374,6 +384,10 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0))
or 0
)
image_tokens = (
cast(Optional[int], getattr(usage.prompt_tokens_details, "image_tokens", 0))
or 0
)
character_count = (
cast(
Optional[int],
@ -398,6 +412,7 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
cache_creation_token_details=cache_creation_token_details,
text_tokens=text_tokens,
audio_tokens=audio_tokens,
image_tokens=image_tokens,
character_count=character_count,
image_count=image_count,
video_length_seconds=video_length_seconds,
@ -470,6 +485,11 @@ def _calculate_input_cost(
model_info, "input_cost_per_audio_token", prompt_tokens_details["audio_tokens"]
)
### IMAGE TOKEN COST (for gpt-image-1 and similar models)
prompt_cost += calculate_cost_component(
model_info, "input_cost_per_image_token", prompt_tokens_details["image_tokens"]
)
### CACHE WRITING COST - Now uses tiered pricing
prompt_cost += calculate_cache_writing_cost(
cache_creation_tokens=prompt_tokens_details["cache_creation_tokens"],
@ -533,6 +553,7 @@ def generic_cost_per_token(
cache_creation_token_details=None,
text_tokens=usage.prompt_tokens,
audio_tokens=0,
image_tokens=0,
character_count=0,
image_count=0,
video_length_seconds=0,
@ -583,12 +604,22 @@ def generic_cost_per_token(
reasoning_tokens = completion_tokens_details["reasoning_tokens"]
image_tokens = completion_tokens_details["image_tokens"]
# Only assume all tokens are text if there's NO breakdown at all
# If image_tokens, audio_tokens, or reasoning_tokens exist, respect text_tokens=0
# Handle text_tokens calculation:
# 1. If text_tokens is explicitly provided and > 0, use it
# 2. If there's a breakdown (reasoning/audio/image tokens), calculate text_tokens as the remainder
# 3. If no breakdown at all, assume all completion_tokens are text_tokens
has_token_breakdown = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0
if text_tokens == 0 and not has_token_breakdown:
text_tokens = usage.completion_tokens
is_text_tokens_total = True
if text_tokens == 0:
if has_token_breakdown:
# Calculate text tokens as remainder when we have a breakdown
# This handles cases like OpenAI's reasoning models where text_tokens isn't provided
text_tokens = max(
0, usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens
)
else:
# No breakdown at all, all tokens are text tokens
text_tokens = usage.completion_tokens
is_text_tokens_total = True
## TEXT COST
completion_cost = float(text_tokens) * completion_base_cost
@ -782,6 +813,50 @@ class CostCalculatorUtils:
model=model,
image_response=completion_response,
)
elif custom_llm_provider == litellm.LlmProviders.OPENAI.value:
# Check if this is a gpt-image model (token-based pricing)
model_lower = model.lower()
if "gpt-image-1" in model_lower:
from litellm.llms.openai.image_generation.cost_calculator import (
cost_calculator as openai_gpt_image_cost_calculator,
)
return openai_gpt_image_cost_calculator(
model=model,
image_response=completion_response,
custom_llm_provider=custom_llm_provider,
)
# Fall through to default for DALL-E models
return default_image_cost_calculator(
model=model,
quality=quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
optional_params=optional_params,
)
elif custom_llm_provider == litellm.LlmProviders.AZURE.value:
# Check if this is a gpt-image model (token-based pricing)
model_lower = model.lower()
if "gpt-image-1" in model_lower:
from litellm.llms.openai.image_generation.cost_calculator import (
cost_calculator as openai_gpt_image_cost_calculator,
)
return openai_gpt_image_cost_calculator(
model=model,
image_response=completion_response,
custom_llm_provider=custom_llm_provider,
)
# Fall through to default for DALL-E models
return default_image_cost_calculator(
model=model,
quality=quality,
custom_llm_provider=custom_llm_provider,
n=n,
size=size,
optional_params=optional_params,
)
else:
return default_image_cost_calculator(
model=model,

46
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -445,25 +445,43 @@ def convert_to_model_response_object( # noqa: PLR0915
hidden_params["additional_headers"] = additional_headers
### CHECK IF ERROR IN RESPONSE ### - openrouter returns these in the dictionary
# Some OpenAI-compatible providers (e.g., Apertis) return empty error objects
# even on success. Only raise if the error contains meaningful data.
if (
response_object is not None
and "error" in response_object
and response_object["error"] is not None
):
error_args = {"status_code": 422, "message": "Error in response object"}
if isinstance(response_object["error"], dict):
if "code" in response_object["error"]:
error_args["status_code"] = response_object["error"]["code"]
if "message" in response_object["error"]:
if isinstance(response_object["error"]["message"], dict):
message_str = json.dumps(response_object["error"]["message"])
else:
message_str = str(response_object["error"]["message"])
error_args["message"] = message_str
raised_exception = Exception()
setattr(raised_exception, "status_code", error_args["status_code"])
setattr(raised_exception, "message", error_args["message"])
raise raised_exception
error_obj = response_object["error"]
has_meaningful_error = False
if isinstance(error_obj, dict):
# Check if error dict has non-empty message or non-null code
error_message = error_obj.get("message", "")
error_code = error_obj.get("code")
has_meaningful_error = bool(error_message) or error_code is not None
elif isinstance(error_obj, str):
# String error is meaningful if non-empty
has_meaningful_error = bool(error_obj)
else:
# Any other truthy value is considered meaningful
has_meaningful_error = True
if has_meaningful_error:
error_args = {"status_code": 422, "message": "Error in response object"}
if isinstance(error_obj, dict):
if "code" in error_obj:
error_args["status_code"] = error_obj["code"]
if "message" in error_obj:
if isinstance(error_obj["message"], dict):
message_str = json.dumps(error_obj["message"])
else:
message_str = str(error_obj["message"])
error_args["message"] = message_str
raised_exception = Exception()
setattr(raised_exception, "status_code", error_args["status_code"])
setattr(raised_exception, "message", error_args["message"])
raise raised_exception
try:
if response_type == "completion" and (

3
litellm/litellm_core_utils/logging_callback_manager.py
View file

@ -166,6 +166,7 @@ class LoggingCallbackManager:
endpoint = callback_config.get("endpoint")
headers = callback_config.get("headers")
event_types = callback_config.get("event_types")
log_format = callback_config.get("log_format")
if endpoint is None or headers is None:
verbose_logger.warning(
@ -180,6 +181,7 @@ class LoggingCallbackManager:
and cached_logger.endpoint == endpoint
and cached_logger.headers == headers
and cached_logger.event_types == event_types
and cached_logger.log_format == log_format
):
return cached_logger
@ -187,6 +189,7 @@ class LoggingCallbackManager:
endpoint=endpoint,
headers=headers,
event_types=event_types,
log_format=log_format,
)
_generic_api_logger_cache[callback] = new_logger
return new_logger

96
litellm/litellm_core_utils/logging_worker.py
View file

@ -51,6 +51,7 @@ class LoggingWorker:
self._worker_task: Optional[asyncio.Task] = None
self._running_tasks: set[asyncio.Task] = set()
self._sem: Optional[asyncio.Semaphore] = None
self._bound_loop: Optional[asyncio.AbstractEventLoop] = None
self._last_aggressive_clear_time: float = 0.0
self._aggressive_clear_in_progress: bool = False
@ -58,9 +59,27 @@ class LoggingWorker:
atexit.register(self._flush_on_exit)
def _ensure_queue(self) -> None:
"""Initialize the queue if it doesn't exist."""
"""Initialize the queue if it doesn't exist or if event loop has changed."""
try:
current_loop = asyncio.get_running_loop()
except RuntimeError:
# No running loop, can't initialize
return
# Check if we need to reinitialize due to event loop change
if self._queue is not None and self._bound_loop is not current_loop:
verbose_logger.debug(
"LoggingWorker: Event loop changed, reinitializing queue and worker"
)
# Clear old state - these are bound to the old loop
self._queue = None
self._sem = None
self._worker_task = None
self._running_tasks.clear()
if self._queue is None:
self._queue = asyncio.Queue(maxsize=self.max_queue_size)
self._bound_loop = current_loop
def start(self) -> None:
"""Start the logging worker. Idempotent - safe to call multiple times."""
@ -126,7 +145,7 @@ class LoggingWorker:
# Capture the current context when enqueueing
task = LoggingTask(coroutine=coroutine, context=contextvars.copy_context())
try:
self._queue.put_nowait(task)
except asyncio.QueueFull:
@ -141,15 +160,15 @@ class LoggingWorker:
"""
if self._aggressive_clear_in_progress:
return False
try:
loop = asyncio.get_running_loop()
current_time = loop.time()
time_since_last_clear = current_time - self._last_aggressive_clear_time
if time_since_last_clear < LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS:
return False
return True
except RuntimeError:
# No event loop running, drop the task
@ -158,8 +177,8 @@ class LoggingWorker:
def _mark_aggressive_clear_started(self) -> None:
"""
Mark that an aggressive clear operation has started.
Note: This should only be called after _should_start_aggressive_clear()
Note: This should only be called after _should_start_aggressive_clear()
returns True, which guarantees an event loop exists.
"""
loop = asyncio.get_running_loop()
@ -171,7 +190,7 @@ class LoggingWorker:
Handle queue full condition by either starting an aggressive clear
or scheduling a delayed retry.
"""
if self._should_start_aggressive_clear():
self._mark_aggressive_clear_started()
# Schedule clearing as async task so enqueue returns immediately (non-blocking)
@ -191,7 +210,8 @@ class LoggingWorker:
time_since_last_clear = current_time - self._last_aggressive_clear_time
remaining_cooldown = max(
0.0,
LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS - time_since_last_clear
LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS
- time_since_last_clear,
)
# Add a small buffer (10% of cooldown or 50ms, whichever is larger) to ensure
# cooldown has expired and aggressive clear has completed
@ -212,7 +232,7 @@ class LoggingWorker:
# Check that we have a running event loop (will raise RuntimeError if not)
asyncio.get_running_loop()
delay = self._calculate_retry_delay()
# Schedule the retry as a background task
asyncio.create_task(self._retry_enqueue_task(task, delay))
except RuntimeError:
@ -225,11 +245,11 @@ class LoggingWorker:
This is called as a background task from _schedule_delayed_enqueue_retry.
"""
await asyncio.sleep(delay)
# Try to enqueue the task directly, preserving its original context
if self._queue is None:
return
try:
self._queue.put_nowait(task)
except asyncio.QueueFull:
@ -243,15 +263,17 @@ class LoggingWorker:
"""
if self._queue is None:
return []
# Calculate items based on percentage of queue size
items_to_extract = (self.max_queue_size * LOGGING_WORKER_CLEAR_PERCENTAGE) // 100
items_to_extract = (
self.max_queue_size * LOGGING_WORKER_CLEAR_PERCENTAGE
) // 100
# Use actual queue size to avoid unnecessary iterations
actual_size = self._queue.qsize()
if actual_size == 0:
return []
items_to_extract = min(items_to_extract, actual_size)
# Extract tasks from queue (using list comprehension would require wrapping in try/except)
extracted_tasks = []
for _ in range(items_to_extract):
@ -259,10 +281,12 @@ class LoggingWorker:
extracted_tasks.append(self._queue.get_nowait())
except asyncio.QueueEmpty:
break
return extracted_tasks
async def _aggressively_clear_queue_async(self, new_task: Optional[LoggingTask] = None) -> None:
async def _aggressively_clear_queue_async(
self, new_task: Optional[LoggingTask] = None
) -> None:
"""
Aggressively clear the queue by extracting and processing items.
This is called when the queue is full to prevent dropping logs.
@ -271,18 +295,20 @@ class LoggingWorker:
try:
if self._queue is None:
return
extracted_tasks = self._extract_tasks_from_queue()
# Add new task to extracted tasks to process directly
if new_task is not None:
extracted_tasks.append(new_task)
# Process extracted tasks directly
if extracted_tasks:
await self._process_extracted_tasks(extracted_tasks)
except Exception as e:
verbose_logger.exception(f"LoggingWorker error during aggressive clear: {e}")
verbose_logger.exception(
f"LoggingWorker error during aggressive clear: {e}"
)
finally:
# Always reset the flag even if an error occurs
self._aggressive_clear_in_progress = False
@ -291,7 +317,7 @@ class LoggingWorker:
"""Process a single task and mark it done."""
if self._queue is None:
return
try:
await asyncio.wait_for(
task["context"].run(asyncio.create_task, task["coroutine"]),
@ -310,7 +336,7 @@ class LoggingWorker:
"""
if not tasks or self._queue is None:
return
# Process all tasks concurrently for maximum speed
await asyncio.gather(*[self._process_single_task(task) for task in tasks])
@ -361,10 +387,7 @@ class LoggingWorker:
for _ in range(MAX_ITERATIONS_TO_CLEAR_QUEUE):
# Check if we've exceeded the maximum time
if (
asyncio.get_event_loop().time() - start_time
>= MAX_TIME_TO_CLEAR_QUEUE
):
if asyncio.get_event_loop().time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE:
verbose_logger.warning(
f"clear_queue exceeded max_time of {MAX_TIME_TO_CLEAR_QUEUE}s, stopping early"
)
@ -381,6 +404,9 @@ class LoggingWorker:
except Exception:
# Suppress errors during cleanup
pass
finally:
# Clear reference to prevent memory leaks
task = None
self._queue.task_done() # If you're using join() elsewhere
except asyncio.QueueEmpty:
break
@ -410,7 +436,7 @@ class LoggingWorker:
This ensures callbacks queued by async completions are processed
even when the script exits before the worker loop can handle them.
Note: All logging in this method is wrapped to handle cases where
logging handlers are closed during shutdown.
"""
@ -423,7 +449,9 @@ class LoggingWorker:
return
queue_size = self._queue.qsize()
self._safe_log("info", f"[LoggingWorker] atexit: Flushing {queue_size} remaining events...")
self._safe_log(
"info", f"[LoggingWorker] atexit: Flushing {queue_size} remaining events..."
)
# Create a new event loop since the original is closed
loop = asyncio.new_event_loop()
@ -438,7 +466,7 @@ class LoggingWorker:
if loop.time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE:
self._safe_log(
"warning",
f"[LoggingWorker] atexit: Reached time limit ({MAX_TIME_TO_CLEAR_QUEUE}s), stopping flush"
f"[LoggingWorker] atexit: Reached time limit ({MAX_TIME_TO_CLEAR_QUEUE}s), stopping flush",
)
break
@ -456,8 +484,14 @@ class LoggingWorker:
except Exception:
# Silent failure to not break user's program
pass
finally:
# Clear reference to prevent memory leaks
task = None
self._safe_log("info", f"[LoggingWorker] atexit: Successfully flushed {processed} events!")
self._safe_log(
"info",
f"[LoggingWorker] atexit: Successfully flushed {processed} events!",
)
finally:
loop.close()

32
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -1087,9 +1087,35 @@ def _parse_content_for_reasoning(
return None, message_text
def _extract_base64_data(image_url: str) -> str:
"""
Extract pure base64 data from an image URL.
If the URL is a data URL (e.g., "data:image/png;base64,iVBOR..."),
extract and return only the base64 data portion.
Otherwise, return the original URL unchanged.
This is needed for providers like Ollama that expect pure base64 data
rather than full data URLs.
Args:
image_url: The image URL or data URL to process
Returns:
The base64 data if it's a data URL, otherwise the original URL
"""
if image_url.startswith("data:") and ";base64," in image_url:
return image_url.split(";base64,", 1)[1]
return image_url
def extract_images_from_message(message: AllMessageValues) -> List[str]:
"""
Extract images from a message
Extract images from a message.
For data URLs (e.g., "data:image/png;base64,iVBOR..."), only the base64
data portion is extracted. This is required for providers like Ollama
that expect pure base64 data rather than full data URLs.
"""
images = []
message_content = message.get("content")
@ -1098,7 +1124,7 @@ def extract_images_from_message(message: AllMessageValues) -> List[str]:
image_url = m.get("image_url")
if image_url:
if isinstance(image_url, str):
images.append(image_url)
images.append(_extract_base64_data(image_url))
elif isinstance(image_url, dict) and "url" in image_url:
images.append(image_url["url"])
images.append(_extract_base64_data(image_url["url"]))
return images

15
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -930,7 +930,8 @@ def create_anthropic_image_param(
# Check if the image URL is an HTTP/HTTPS URL
if image_url.startswith("http://") or image_url.startswith("https://"):
# For Bedrock invoke, always convert URLs to base64 (Bedrock invoke doesn't support URLs)
# For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
# as these providers don't support URL sources for images
if is_bedrock_invoke or image_url.startswith("http://"):
base64_url = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(
@ -1496,9 +1497,10 @@ def convert_to_gemini_tool_call_result(
content_type = content.get("type", "")
if content_type == "text":
content_str += content.get("text", "")
elif content_type == "input_image":
# Extract image for inline_data (for Computer Use screenshots)
image_url = content.get("image_url", "")
elif content_type in ("input_image", "image_url"):
# Extract image for inline_data (for Computer Use screenshots and tool results)
image_url_data = content.get("image_url", "")
image_url = image_url_data.get("url", "") if isinstance(image_url_data, dict) else image_url_data
if image_url:
# Convert image to base64 blob format for Gemini
@ -2022,9 +2024,12 @@ def anthropic_messages_pt( # noqa: PLR0915
"format": image_url_value.get("format"),
}
# Bedrock invoke models have format: invoke/...
# Vertex AI Anthropic also doesn't support URL sources for images
is_bedrock_invoke = model.lower().startswith("invoke/")
is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
force_base64 = is_bedrock_invoke or is_vertex_ai
_anthropic_content_element = create_anthropic_image_param(
image_url_input, format=format, is_bedrock_invoke=is_bedrock_invoke
image_url_input, format=format, is_bedrock_invoke=force_base64
)
_content_element = add_cache_control_to_content(
anthropic_content_element=_anthropic_content_element,

11
litellm/llms/anthropic/chat/transformation.py
View file

@ -54,7 +54,10 @@ from litellm.types.utils import (
CompletionTokensDetailsWrapper,
)
from litellm.types.utils import Message as LitellmMessage
from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse
from litellm.types.utils import (
PromptTokensDetailsWrapper,
ServerToolUse,
)
from litellm.utils import (
ModelResponse,
Usage,
@ -204,9 +207,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
) # Relevant issue: https://github.com/BerriAI/litellm/issues/7755
def get_cache_control_headers(self) -> dict:
# Anthropic no longer requires the prompt-caching beta header
# Prompt caching now works automatically when cache_control is used in messages
# Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
return {
"anthropic-version": "2023-06-01",
"anthropic-beta": "prompt-caching-2024-07-31",
}
def _map_tool_choice(
@ -1034,7 +1039,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_messages = anthropic_messages_pt(
model=model,
messages=messages,
llm_provider="anthropic",
llm_provider=self.custom_llm_provider or "anthropic",
)
except Exception as e:
raise AnthropicError(

16
litellm/llms/anthropic/common_utils.py
View file

@ -12,7 +12,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
)
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import AllAnthropicToolsValues, AnthropicMcpServerTool, ANTHROPIC_HOSTED_TOOLS
from litellm.types.llms.anthropic import (
ANTHROPIC_HOSTED_TOOLS,
AllAnthropicToolsValues,
AnthropicMcpServerTool,
)
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import TokenCountResponse
@ -273,8 +277,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
beta_header = self.get_computer_tool_beta_header(computer_tool_used)
betas.append(beta_header)
if prompt_caching_set:
betas.append("prompt-caching-2024-07-31")
# Anthropic no longer requires the prompt-caching beta header
# Prompt caching now works automatically when cache_control is used in messages
# Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
if file_id_used:
betas.append("files-api-2025-04-14")
@ -305,8 +310,9 @@ class AnthropicModelInfo(BaseLLMModelInfo):
container_with_skills_used: bool = False,
) -> dict:
betas = set()
if prompt_caching_set:
betas.add("prompt-caching-2024-07-31")
# Anthropic no longer requires the prompt-caching beta header
# Prompt caching now works automatically when cache_control is used in messages
# Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
if computer_tool_used:
beta_header = self.get_computer_tool_beta_header(computer_tool_used)
betas.add(beta_header)

7
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -48,7 +48,12 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
headers = BaseAzureLLM._base_validate_azure_environment(
headers=headers, litellm_params=litellm_params_obj
)
# Azure Anthropic uses x-api-key header (not api-key)
# Convert api-key to x-api-key if present
if "api-key" in headers and "x-api-key" not in headers:
headers["x-api-key"] = headers.pop("api-key")
# Set anthropic-version header
if "anthropic-version" not in headers:
headers["anthropic-version"] = "2023-06-01"

4
litellm/llms/gemini/google_genai/transformation.py
View file

@ -312,7 +312,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
)
request_dict = cast(dict, typed_generate_content_request)
if system_instruction is not None:
request_dict["systemInstruction"] = system_instruction
return request_dict
def transform_generate_content_response(

2
litellm/llms/ollama/completion/handler.py
View file

@ -15,7 +15,7 @@ def _prepare_ollama_embedding_payload(
) -> Dict[str, Any]:
data: Dict[str, Any] = {"model": model, "input": prompts}
special_optional_params = ["truncate", "options", "keep_alive"]
special_optional_params = ["truncate", "options", "keep_alive","dimensions"]
for k, v in optional_params.items():
if k in special_optional_params:

63
litellm/llms/openai/image_generation/cost_calculator.py Normal file
View file

@ -0,0 +1,63 @@
"""
Cost calculator for OpenAI image generation models (gpt-image-1, gpt-image-1-mini)
These models use token-based pricing instead of pixel-based pricing like DALL-E.
"""
from typing import Optional
from litellm import verbose_logger
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.utils import ImageResponse
def cost_calculator(
model: str,
image_response: ImageResponse,
custom_llm_provider: Optional[str] = None,
) -> float:
"""
Calculate cost for OpenAI gpt-image-1 and gpt-image-1-mini models.
Uses the same usage format as Responses API, so we reuse the helper
to transform to chat completion format and use generic_cost_per_token.
Args:
model: The model name (e.g., "gpt-image-1", "gpt-image-1-mini")
image_response: The ImageResponse containing usage data
custom_llm_provider: Optional provider name
Returns:
float: Total cost in USD
"""
usage = getattr(image_response, "usage", None)
if usage is None:
verbose_logger.debug(
f"No usage data available for {model}, cannot calculate token-based cost"
)
return 0.0
# Transform ImageUsage to Usage using the existing helper
# ImageUsage has the same format as ResponseAPIUsage
chat_usage = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(
usage
)
# Use generic_cost_per_token for cost calculation
prompt_cost, completion_cost = generic_cost_per_token(
model=model,
usage=chat_usage,
custom_llm_provider=custom_llm_provider or "openai",
)
total_cost = prompt_cost + completion_cost
verbose_logger.debug(
f"OpenAI gpt-image cost calculation for {model}: "
f"prompt_cost=${prompt_cost:.6f}, completion_cost=${completion_cost:.6f}, "
f"total=${total_cost:.6f}"
)
return total_cost

4
litellm/llms/sap/chat/transformation.py
View file

@ -203,9 +203,13 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig):
headers: dict,
) -> dict:
supported_params = self.get_supported_openai_params(model)
# Include extra params that passed validation (e.g., thinking_config for Gemini models via allowed_openai_params)
extra_params = [k for k in optional_params if k not in supported_params and k not in {"tools", "model_version"}]
supported_params = supported_params + extra_params
model_params = {
k: v for k, v in optional_params.items() if k in supported_params
}
model_version = optional_params.pop("model_version", "latest")
template = []
for message in messages:

13
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -383,7 +383,18 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915
and isinstance(_message_content, str)
):
assistant_text = _message_content
assistant_content.append(PartType(text=assistant_text)) # type: ignore
# Check if message has thought_signatures in provider_specific_fields
provider_specific_fields = assistant_msg.get("provider_specific_fields")
thought_signatures = None
if provider_specific_fields and isinstance(provider_specific_fields, dict):
thought_signatures = provider_specific_fields.get("thought_signatures")
# If we have thought signatures, add them to the part
if thought_signatures and isinstance(thought_signatures, list) and len(thought_signatures) > 0:
# Use the first signature for the text part (Gemini expects one signature per part)
assistant_content.append(PartType(text=assistant_text, thoughtSignature=thought_signatures[0])) # type: ignore
else:
assistant_content.append(PartType(text=assistant_text)) # type: ignore
## HANDLE ASSISTANT FUNCTION CALL
if (

78
litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
View file

@ -552,24 +552,46 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"Invalid tool={}. Use `litellm.set_verbose` or `litellm --detailed_debug` to see raw request."
)
# Only include function_declarations if there are actual functions
_tools = Tools()
# Build list of Tool objects - each Tool should contain exactly one type
# per Vertex AI API spec: "A Tool object should contain exactly one type of Tool"
_tools_list: List[Tools] = []
# Function declarations can be grouped together in one Tool
if gtool_func_declarations:
_tools["function_declarations"] = gtool_func_declarations
func_tool = Tools()
func_tool["function_declarations"] = gtool_func_declarations
_tools_list.append(func_tool)
# Each special tool type must be in its own Tool object
if googleSearch is not None:
_tools[VertexToolName.GOOGLE_SEARCH.value] = googleSearch
search_tool = Tools()
search_tool[VertexToolName.GOOGLE_SEARCH.value] = googleSearch
_tools_list.append(search_tool)
if googleSearchRetrieval is not None:
_tools[VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value] = googleSearchRetrieval
retrieval_tool = Tools()
retrieval_tool[VertexToolName.GOOGLE_SEARCH_RETRIEVAL.value] = googleSearchRetrieval
_tools_list.append(retrieval_tool)
if enterpriseWebSearch is not None:
_tools[VertexToolName.ENTERPRISE_WEB_SEARCH.value] = enterpriseWebSearch
enterprise_tool = Tools()
enterprise_tool[VertexToolName.ENTERPRISE_WEB_SEARCH.value] = enterpriseWebSearch
_tools_list.append(enterprise_tool)
if code_execution is not None:
_tools[VertexToolName.CODE_EXECUTION.value] = code_execution
code_tool = Tools()
code_tool[VertexToolName.CODE_EXECUTION.value] = code_execution
_tools_list.append(code_tool)
if urlContext is not None:
_tools[VertexToolName.URL_CONTEXT.value] = urlContext
url_tool = Tools()
url_tool[VertexToolName.URL_CONTEXT.value] = urlContext
_tools_list.append(url_tool)
if googleMaps is not None:
_tools[VertexToolName.GOOGLE_MAPS.value] = googleMaps
maps_tool = Tools()
maps_tool[VertexToolName.GOOGLE_MAPS.value] = googleMaps
_tools_list.append(maps_tool)
if computerUse is not None:
_tools[VertexToolName.COMPUTER_USE.value] = computerUse
computer_tool = Tools()
computer_tool[VertexToolName.COMPUTER_USE.value] = computerUse
_tools_list.append(computer_tool)
# Add retrieval config to toolConfig if googleMaps has location data
if google_maps_retrieval_config is not None:
@ -579,7 +601,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
"retrievalConfig"
] = google_maps_retrieval_config
return [_tools]
return _tools_list
def _map_response_schema(self, value: dict) -> dict:
old_schema = deepcopy(value)
@ -1210,6 +1232,25 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
thinking_blocks.append(block)
return thinking_blocks
def _extract_thought_signatures_from_parts(
self, parts: List[HttpxPartType]
) -> Optional[List[str]]:
"""Extract thoughtSignature values from parts.
Per Google's docs, thoughtSignature is returned for multi-turn context preservation
and can appear on parts even without thought: true (e.g., regular text responses,
function calls). This method extracts all thoughtSignature values from parts.
Returns:
List of thoughtSignature strings if any are found, None otherwise
"""
signatures: List[str] = []
for part in parts:
signature = part.get("thoughtSignature")
if signature is not None:
signatures.append(signature)
return signatures if signatures else None
def _extract_image_response_from_parts(
self, parts: List[HttpxPartType]
) -> Optional[List[ImageURLListItem]]:
@ -1620,6 +1661,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
from litellm.types.utils import Delta, StreamingChoices
annotations = chat_completion_message.get("annotations") # type: ignore
provider_specific_fields = chat_completion_message.get("provider_specific_fields") # type: ignore
# create a streaming choice object
choice = StreamingChoices(
finish_reason=VertexGeminiConfig._check_finish_reason(
@ -1633,6 +1675,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
images=image_response,
function_call=functions,
annotations=annotations, # type: ignore
provider_specific_fields=provider_specific_fields,
),
logprobs=chat_completion_logprobs,
enhancements=None,
@ -1811,6 +1854,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
)
)
# Extract thoughtSignatures from parts (can exist without thought: true)
thought_signatures = (
VertexGeminiConfig()._extract_thought_signatures_from_parts(
parts=candidate["content"]["parts"]
)
)
if audio_response is not None:
cast(Dict[str, Any], chat_completion_message)[
"audio"
@ -1876,6 +1926,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
reasoning_content = "\n".join(reasoning_content_parts)
chat_completion_message["reasoning_content"] = reasoning_content
# Store thoughtSignatures in provider_specific_fields
if thought_signatures is not None:
if "provider_specific_fields" not in chat_completion_message:
chat_completion_message["provider_specific_fields"] = {}
chat_completion_message["provider_specific_fields"]["thought_signatures"] = thought_signatures # type: ignore
if isinstance(model_response, ModelResponseStream):
choice = VertexGeminiConfig._create_streaming_choice(
chat_completion_message=chat_completion_message,

11
litellm/llms/zai/chat/transformation.py
View file

@ -20,7 +20,7 @@ class ZAIChatConfig(OpenAIGPTConfig):
return api_base, dynamic_api_key
def get_supported_openai_params(self, model: str) -> list:
return [
base_params = [
"max_tokens",
"stream",
"stream_options",
@ -31,3 +31,12 @@ class ZAIChatConfig(OpenAIGPTConfig):
"tool_choice",
]
import litellm
try:
if litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider):
base_params.append("thinking")
except Exception:
pass
return base_params

452
litellm/model_prices_and_context_window_backup.json
View file

@ -249,6 +249,30 @@
"/v1/images/generations"
]
},
"aiml/google/imagen-4.0-ultra-generate-001": {
"litellm_provider": "aiml",
"metadata": {
"notes": "Imagen 4.0 Ultra Generate API - Photorealistic image generation with precise text rendering"
},
"mode": "image_generation",
"output_cost_per_image": 0.063,
"source": "https://docs.aimlapi.com/api-references/image-models/google/imagen-4-ultra-generate",
"supported_endpoints": [
"/v1/images/generations"
]
},
"aiml/google/nano-banana-pro": {
"litellm_provider": "aiml",
"metadata": {
"notes": "Gemini 3 Pro Image (Nano Banana Pro) - Advanced text-to-image generation with reasoning and 4K resolution support"
},
"mode": "image_generation",
"output_cost_per_image": 0.1575,
"source": "https://docs.aimlapi.com/api-references/image-models/google/gemini-3-pro-image-preview",
"supported_endpoints": [
"/v1/images/generations"
]
},
"amazon.nova-canvas-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 2600,
@ -3508,6 +3532,40 @@
"supports_service_tier": true,
"supports_vision": true
},
"azure/gpt-5.2-chat": {
"cache_read_input_token_cost": 1.75e-07,
"cache_read_input_token_cost_priority": 3.5e-07,
"input_cost_per_token": 1.75e-06,
"input_cost_per_token_priority": 3.5e-06,
"litellm_provider": "azure",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_token": 1.4e-05,
"output_cost_per_token_priority": 2.8e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"azure/gpt-5.2-chat-2025-12-11": {
"cache_read_input_token_cost": 1.75e-07,
"cache_read_input_token_cost_priority": 3.5e-07,
@ -3605,12 +3663,16 @@
"supports_web_search": true
},
"azure/gpt-image-1": {
"input_cost_per_pixel": 4.0054321e-08,
"cache_read_input_image_token_cost": 2.5e-06,
"cache_read_input_token_cost": 1.25e-06,
"input_cost_per_image_token": 1e-05,
"input_cost_per_token": 5e-06,
"litellm_provider": "azure",
"mode": "image_generation",
"output_cost_per_pixel": 0.0,
"output_cost_per_image_token": 4e-05,
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"azure/hd/1024-x-1024/dall-e-3": {
@ -3713,12 +3775,16 @@
]
},
"azure/gpt-image-1-mini": {
"input_cost_per_pixel": 8.0566406e-09,
"cache_read_input_image_token_cost": 2.5e-07,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_image_token": 2.5e-06,
"input_cost_per_token": 2e-06,
"litellm_provider": "azure",
"mode": "image_generation",
"output_cost_per_pixel": 0.0,
"output_cost_per_image_token": 8e-06,
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"azure/gpt-image-1.5": {
@ -10885,13 +10951,13 @@
"supports_tool_choice": true
},
"fireworks_ai/accounts/fireworks/models/deepseek-v3p2": {
"input_cost_per_token": 1.2e-06,
"input_cost_per_token": 5.6e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"output_cost_per_token": 1.68e-06,
"source": "https://fireworks.ai/models/fireworks/deepseek-v3p2",
"supports_function_calling": true,
"supports_reasoning": true,
@ -16922,6 +16988,336 @@
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.034,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.133,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.034,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.133,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"gpt-5": {
"cache_read_input_token_cost": 1.25e-07,
"cache_read_input_token_cost_flex": 6.25e-08,
@ -17683,16 +18079,16 @@
"supports_vision": true
},
"gpt-image-1": {
"input_cost_per_image": 0.042,
"input_cost_per_pixel": 4.0054321e-08,
"input_cost_per_token": 0.000005,
"input_cost_per_image_token": 0.00001,
"cache_read_input_image_token_cost": 2.5e-06,
"cache_read_input_token_cost": 1.25e-06,
"input_cost_per_image_token": 1e-05,
"input_cost_per_token": 5e-06,
"litellm_provider": "openai",
"mode": "image_generation",
"output_cost_per_pixel": 0.0,
"output_cost_per_token": 0.00004,
"output_cost_per_image_token": 4e-05,
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"gpt-image-1-mini": {
@ -18117,6 +18513,18 @@
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/gemma-7b-it": {
"input_cost_per_token": 5e-08,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 8e-08,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/meta-llama/llama-guard-4-12b": {
"input_cost_per_token": 2e-07,
"litellm_provider": "groq",
@ -29241,6 +29649,20 @@
"supports_vision": true,
"supports_web_search": true
},
"zai/glm-4.7": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 1.1e-07,
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.2e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-4.6": {
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.2e-06,

340
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -11,7 +11,7 @@ import datetime
import hashlib
import json
import re
from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, cast
from urllib.parse import urlparse
from fastapi import HTTPException
@ -536,9 +536,12 @@ class MCPServerManager:
client_secret=client_secret_value
or getattr(mcp_server, "client_secret", None),
scopes=resolved_scopes,
authorization_url=getattr(mcp_oauth_metadata, "authorization_url", None),
token_url=getattr(mcp_oauth_metadata, "token_url", None),
registration_url=getattr(mcp_oauth_metadata, "registration_url", None),
authorization_url=mcp_server.authorization_url
or getattr(mcp_oauth_metadata, "authorization_url", None),
token_url=mcp_server.token_url
or getattr(mcp_oauth_metadata, "token_url", None),
registration_url=mcp_server.registration_url
or getattr(mcp_oauth_metadata, "registration_url", None),
command=getattr(mcp_server, "command", None),
args=getattr(mcp_server, "args", None) or [],
env=env_dict,
@ -548,7 +551,7 @@ class MCPServerManager:
)
return new_server
async def add_update_server(self, mcp_server: LiteLLM_MCPServerTable):
async def add_server(self, mcp_server: LiteLLM_MCPServerTable):
try:
if mcp_server.server_id not in self.registry:
new_server = await self.build_mcp_server_from_table(mcp_server)
@ -559,6 +562,17 @@ class MCPServerManager:
verbose_logger.debug(f"Failed to add MCP server: {str(e)}")
raise e
async def update_server(self, mcp_server: LiteLLM_MCPServerTable):
try:
if mcp_server.server_id in self.registry:
new_server = await self.build_mcp_server_from_table(mcp_server)
self.registry[mcp_server.server_id] = new_server
verbose_logger.debug(f"Updated MCP Server: {new_server.name}")
except Exception as e:
verbose_logger.debug(f"Failed to udpate MCP server: {str(e)}")
raise e
def get_all_mcp_server_ids(self) -> Set[str]:
"""
Get all MCP server IDs
@ -2040,7 +2054,7 @@ class MCPServerManager:
verbose_logger.debug(
f"Adding server to registry: {server.server_id} ({server.server_name})"
)
await self.add_update_server(server)
await self.add_server(server)
verbose_logger.debug(
f"Registry now contains {len(self.get_registry())} servers"
@ -2127,7 +2141,7 @@ class MCPServerManager:
async def health_check_server(
self, server_id: str, mcp_auth_header: Optional[str] = None
) -> Dict[str, Any]:
) -> LiteLLM_MCPServerTable:
"""
Perform a health check on a specific MCP server.
@ -2138,206 +2152,186 @@ class MCPServerManager:
Returns:
Dict containing health check results
"""
import time
from datetime import datetime
server = self.get_mcp_server_by_id(server_id)
if not server:
return {
"server_id": server_id,
"server_name": None,
"status": "unknown",
"error": "Server not found",
"last_health_check": datetime.now().isoformat(),
"response_time_ms": None,
}
start_time = time.time()
try:
# Try to get tools from the server as a health check
tools = await self._get_tools_from_server(server, mcp_auth_header)
response_time = (time.time() - start_time) * 1000
return {
"server_id": server_id,
"server_name": server.name,
"status": "healthy",
"tools_count": len(tools),
"last_health_check": datetime.now().isoformat(),
"response_time_ms": round(response_time, 2),
"error": None,
}
except Exception as e:
response_time = (time.time() - start_time) * 1000
error_message = str(e)
return {
"server_id": server_id,
"server_name": server.name,
"status": "unhealthy",
"last_health_check": datetime.now().isoformat(),
"response_time_ms": round(response_time, 2),
"error": error_message,
}
async def health_check_all_servers(
self, mcp_auth_header: Optional[str] = None
) -> Dict[str, Any]:
"""
Perform health checks on all MCP servers.
Args:
mcp_auth_header: Optional authentication header for the MCP servers
Returns:
Dict containing health check results for all servers
"""
all_servers = self.get_registry()
results = {}
for server_id, server in all_servers.items():
results[server_id] = await self.health_check_server(
server_id, mcp_auth_header
verbose_logger.warning(f"MCP Server {server_id} not found")
return LiteLLM_MCPServerTable(
server_id=server_id,
server_name=None,
transport=MCPTransport.http, # Default transport for not found servers
status="unknown",
health_check_error="Server not found",
last_health_check=datetime.now(),
)
return results
status: Literal["healthy", "unhealthy", "unknown"] = "unknown"
health_check_error = None
async def health_check_allowed_servers(
self,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
mcp_auth_header: Optional[str] = None,
) -> Dict[str, Any]:
"""
Perform health checks on all MCP servers that the user has access to.
# Check if we should skip health check based on auth configuration
should_skip_health_check = False
Args:
user_api_key_auth: User authentication info for access control
mcp_auth_header: Optional authentication header for the MCP servers
# Skip if auth_type is oauth2
if server.auth_type == MCPAuth.oauth2:
should_skip_health_check = True
# Skip if auth_type is not none and authentication_token is missing
elif (
server.auth_type
and server.auth_type != MCPAuth.none
and not server.authentication_token
):
should_skip_health_check = True
Returns:
Dict containing health check results for accessible servers
"""
# Get allowed servers for the user
allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth)
if not should_skip_health_check:
extra_headers = {}
if server.static_headers:
extra_headers.update(server.static_headers)
# Perform health checks on allowed servers
results = {}
for server_id in allowed_server_ids:
results[server_id] = await self.health_check_server(
server_id, mcp_auth_header
client = self._create_mcp_client(
server=server,
mcp_auth_header=None,
extra_headers=extra_headers,
stdio_env=None,
)
return results
try:
async def _noop(session):
return "ok"
# Add timeout wrapper to prevent hanging
await asyncio.wait_for(client.run_with_session(_noop), timeout=10.0)
status = "healthy"
except asyncio.TimeoutError:
health_check_error = "Health check timed out after 10 seconds"
status = "unhealthy"
except Exception as e:
health_check_error = str(e)
status = "unhealthy"
return LiteLLM_MCPServerTable(
server_id=server.server_id,
server_name=server.server_name,
alias=server.alias,
description=(
server.mcp_info.get("description") if server.mcp_info else None
),
url=server.url,
transport=server.transport,
auth_type=server.auth_type,
created_at=datetime.now(),
updated_at=datetime.now(),
teams=[],
mcp_access_groups=server.access_groups or [],
allowed_tools=server.allowed_tools or [],
extra_headers=server.extra_headers or [],
mcp_info=server.mcp_info,
static_headers=server.static_headers,
status=status,
last_health_check=datetime.now(),
health_check_error=health_check_error,
command=getattr(server, "command", None),
args=getattr(server, "args", None) or [],
env=getattr(server, "env", None) or {},
authorization_url=server.authorization_url,
token_url=server.token_url,
registration_url=server.registration_url,
)
async def get_all_mcp_servers_with_health_and_teams(
self,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
include_health: bool = True,
server_ids: Optional[List[str]] = None,
) -> List[LiteLLM_MCPServerTable]:
"""
Get all MCP servers that the user has access to, with health status and team information.
Args:
user_api_key_auth: User authentication info for access control
include_health: Whether to include health check information
server_ids: Optional list of server IDs to filter. If provided, only these servers
will be checked (subject to access control). If None, all accessible servers are checked.
Returns:
List of MCP server objects with health and team data
"""
from litellm.proxy._experimental.mcp_server.db import (
get_all_mcp_servers,
get_mcp_servers,
)
from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
from litellm.proxy.proxy_server import prisma_client
# Get allowed server IDs
allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth)
# Get servers from database
# Filter by requested server_ids if provided
if server_ids:
# Only check servers that are both requested AND accessible
target_server_ids = [sid for sid in server_ids if sid in allowed_server_ids]
else:
# Check all accessible servers
target_server_ids = allowed_server_ids
# Run health checks concurrently
tasks = [self.health_check_server(server_id) for server_id in target_server_ids]
results = await asyncio.gather(*tasks)
# Filter out None results (servers that were not found)
list_mcp_servers = [server for server in results if server is not None]
return list_mcp_servers
async def get_all_allowed_mcp_servers(
self,
user_api_key_auth: Optional[UserAPIKeyAuth] = None,
) -> List[LiteLLM_MCPServerTable]:
"""
Get all MCP servers that the user has access to.
Args:
user_api_key_auth: User authentication info for access control
Returns:
List of MCP server objects without health status
"""
from datetime import datetime
# Get allowed server IDs
allowed_server_ids = await self.get_allowed_mcp_servers(user_api_key_auth)
list_mcp_servers: List[LiteLLM_MCPServerTable] = []
if prisma_client is not None:
list_mcp_servers = await get_mcp_servers(prisma_client, allowed_server_ids)
# If admin, also get all servers from database
if user_api_key_auth and _user_has_admin_view(user_api_key_auth):
all_mcp_servers = await get_all_mcp_servers(prisma_client)
for server in all_mcp_servers:
if server.server_id not in allowed_server_ids:
list_mcp_servers.append(server)
for server_id in allowed_server_ids:
server = self.get_mcp_server_by_id(server_id)
if not server:
verbose_logger.warning(f"MCP Server {server_id} not found in registry")
continue
# Add config.yaml servers
for _server_id, _server_config in self.config_mcp_servers.items():
if _server_id in allowed_server_ids:
list_mcp_servers.append(
LiteLLM_MCPServerTable(
**{
**_server_config.model_dump(),
"created_at": datetime.datetime.now(),
"updated_at": datetime.datetime.now(),
"description": (
_server_config.mcp_info.get("description")
if _server_config.mcp_info
else None
),
"allowed_tools": _server_config.allowed_tools or [],
"mcp_info": _server_config.mcp_info,
"mcp_access_groups": _server_config.access_groups or [],
"extra_headers": _server_config.extra_headers or [],
"command": getattr(_server_config, "command", None),
"args": getattr(_server_config, "args", None) or [],
"env": getattr(_server_config, "env", None) or {},
}
)
)
# Get team information for non-admin users
server_to_teams_map: Dict[str, List[Dict[str, str]]] = {}
if (
user_api_key_auth
and not _user_has_admin_view(user_api_key_auth)
and prisma_client is not None
):
teams = await prisma_client.db.litellm_teamtable.find_many(
include={"object_permission": True}
# Build LiteLLM_MCPServerTable without health check
mcp_server_table = LiteLLM_MCPServerTable(
server_id=server.server_id,
server_name=server.server_name,
alias=server.alias,
description=(
server.mcp_info.get("description") if server.mcp_info else None
),
url=server.url,
transport=server.transport,
auth_type=server.auth_type,
created_at=datetime.now(),
updated_at=datetime.now(),
teams=[],
mcp_access_groups=server.access_groups or [],
allowed_tools=server.allowed_tools or [],
extra_headers=server.extra_headers or [],
mcp_info=server.mcp_info,
static_headers=server.static_headers,
status=None, # No health check performed
last_health_check=None, # No health check performed
health_check_error=None,
command=getattr(server, "command", None),
args=getattr(server, "args", None) or [],
env=getattr(server, "env", None) or {},
authorization_url=server.authorization_url,
token_url=server.token_url,
registration_url=server.registration_url,
)
user_teams = []
for team in teams:
if team.members_with_roles:
for member in team.members_with_roles:
if (
"user_id" in member
and member["user_id"] is not None
and member["user_id"] == user_api_key_auth.user_id
):
user_teams.append(team)
# Create a mapping of server_id to teams that have access to it
for team in user_teams:
if team.object_permission and team.object_permission.mcp_servers:
for server_id in team.object_permission.mcp_servers:
if server_id not in server_to_teams_map:
server_to_teams_map[server_id] = []
server_to_teams_map[server_id].append(
{
"team_id": team.team_id,
"team_alias": team.team_alias,
"organization_id": team.organization_id,
}
)
## mark invalid servers w/ reason for being invalid
valid_server_ids = self.get_all_mcp_server_ids()
for server in list_mcp_servers:
if server.server_id not in valid_server_ids:
server.status = "unhealthy"
## try adding server to registry to get error
try:
await self.add_update_server(server)
except Exception as e:
server.health_check_error = str(e)
server.health_check_error = "Server is not in in memory registry yet. This could be a temporary sync issue."
list_mcp_servers.append(mcp_server_table)
return list_mcp_servers

165
litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py
View file

@ -3,20 +3,118 @@ This module is used to generate MCP tools from OpenAPI specs.
"""
import json
from pathlib import PurePosixPath
from typing import Any, Dict, Optional
import httpx
from urllib.parse import quote
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.proxy._experimental.mcp_server.tool_registry import (
global_mcp_tool_registry,
)
from litellm.types.llms.custom_http import httpxSpecialProvider
# Store the base URL and headers globally
BASE_URL = ""
HEADERS: Dict[str, str] = {}
def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str:
"""Ensure path params cannot introduce directory traversal."""
if param_value is None:
return ""
value_str = str(param_value)
if value_str == "":
return ""
normalized_value = value_str.replace("\\", "/")
if "/" in normalized_value:
raise ValueError(
f"Path parameter '{param_name}' must not contain path separators"
)
if any(part in {".", ".."} for part in PurePosixPath(normalized_value).parts):
raise ValueError(
f"Path parameter '{param_name}' cannot include '.' or '..' segments"
)
return quote(value_str, safe="")
async def _invoke_openapi_operation(
*,
method: str,
base_url: str,
path: str,
headers: Dict[str, str],
path_params: list,
query_params: list,
body_params: list,
provided_params: Dict[str, Any],
) -> str:
"""Execute the OpenAPI operation using provided parameters."""
url = base_url + path
# Replace path parameters with sanitized values
for param_name in path_params:
param_value = provided_params.get(param_name, "")
if param_value:
try:
safe_value = _sanitize_path_parameter_value(param_value, param_name)
except ValueError as exc:
return "Invalid path parameter: " + str(exc)
url = url.replace("{" + param_name + "}", safe_value)
# Build query params
params: Dict[str, Any] = {}
for param_name in query_params:
param_value = provided_params.get(param_name, "")
if param_value:
params[param_name] = param_value
# Build request body
json_body = None
if body_params:
body_value = provided_params.get("body", {})
if isinstance(body_value, dict):
json_body = body_value
elif body_value:
import json as json_module
try:
json_body = (
json_module.loads(body_value)
if isinstance(body_value, str)
else {"data": body_value}
)
except Exception:
json_body = {"data": body_value}
client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
method_lower = method.lower()
if method_lower == "get":
response = await client.get(url, params=params, headers=headers)
elif method_lower == "post":
response = await client.post(
url, params=params, json=json_body, headers=headers
)
elif method_lower == "put":
response = await client.put(url, params=params, json=json_body, headers=headers)
elif method_lower == "delete":
response = await client.delete(url, params=params, headers=headers)
elif method_lower == "patch":
response = await client.patch(
url, params=params, json=json_body, headers=headers
)
else:
return f"Unsupported HTTP method: {method}"
return response.text
def load_openapi_spec(filepath: str) -> Dict[str, Any]:
"""Load OpenAPI specification from JSON file."""
with open(filepath, "r") as f:
@ -135,63 +233,28 @@ def create_tool_function(
func_code = f'''
async def tool_function({params_str}) -> str:
"""Dynamically generated tool function."""
url = base_url + path
# Replace path parameters
path_param_names = {path_params}
for param_name in path_param_names:
param_value = locals().get(param_name, "")
if param_value:
url = url.replace("{{" + param_name + "}}", str(param_value))
# Build query params
query_param_names = {query_params}
params = {{}}
for param_name in query_param_names:
param_value = locals().get(param_name, "")
if param_value:
params[param_name] = param_value
# Build request body
body_param_names = {body_params}
json_body = None
if body_param_names:
body_value = locals().get("body", {{}})
if isinstance(body_value, dict):
json_body = body_value
elif body_value:
# If it's a string, try to parse as JSON
import json as json_module
try:
json_body = json_module.loads(body_value) if isinstance(body_value, str) else {{"data": body_value}}
except:
json_body = {{"data": body_value}}
# Make HTTP request
async with httpx.AsyncClient() as client:
if "{method.lower()}" == "get":
response = await client.get(url, params=params, headers=headers)
elif "{method.lower()}" == "post":
response = await client.post(url, params=params, json=json_body, headers=headers)
elif "{method.lower()}" == "put":
response = await client.put(url, params=params, json=json_body, headers=headers)
elif "{method.lower()}" == "delete":
response = await client.delete(url, params=params, headers=headers)
elif "{method.lower()}" == "patch":
response = await client.patch(url, params=params, json=json_body, headers=headers)
else:
return "Unsupported HTTP method: {method}"
return response.text
provided_params = {{}}
for param_name in {all_params}:
provided_params[param_name] = locals().get(param_name, "")
return await _invoke_openapi_operation(
method="{method}",
base_url=base_url,
path=path,
headers=headers,
path_params={path_params},
query_params={query_params},
body_params={body_params},
provided_params=provided_params,
)
'''
# Execute the function code to create the actual function
local_vars = {
"httpx": httpx,
"headers": headers,
"base_url": base_url,
"path": path,
"method": method,
"_invoke_openapi_operation": _invoke_openapi_operation,
}
exec(func_code, local_vars)

8
litellm/proxy/_experimental/mcp_server/rest_endpoints.py
View file

@ -1,5 +1,4 @@
import importlib
import traceback
from typing import Dict, List, Optional, Union
from fastapi import APIRouter, Depends, Query, Request
@ -347,17 +346,16 @@ if MCP_AVAILABLE:
except Exception as e:
verbose_logger.error(f"Error in MCP operation: {e}", exc_info=True)
stack_trace = traceback.format_exc()
return {
"status": "error",
"message": f"An internal error has occurred: {str(e)}",
"stack_trace": stack_trace,
"message": "An internal error has occurred while testing the MCP server.",
}
@router.post("/test/connection")
@router.post("/test/connection", dependencies=[Depends(user_api_key_auth)])
async def test_connection(
request: Request,
new_mcp_server_request: NewMCPServerRequest,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Test if we can connect to the provided MCP server before adding it

124
litellm/proxy/_types.py
View file

@ -412,7 +412,6 @@ class LiteLLMRoutes(enum.Enum):
agent_routes = [
"/v1/agents",
"/agents",
"/a2a/{agent_id}",
"/a2a/{agent_id}/message/send",
"/a2a/{agent_id}/message/stream",
@ -830,9 +829,9 @@ class GenerateRequestBase(LiteLLMPydanticObjectBase):
allowed_cache_controls: Optional[list] = []
config: Optional[dict] = {}
permissions: Optional[dict] = {}
model_max_budget: Optional[dict] = (
{}
) # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {}
model_max_budget: Optional[
dict
] = {} # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {}
model_config = ConfigDict(protected_namespaces=())
model_rpm_limit: Optional[dict] = None
@ -1035,6 +1034,9 @@ class NewMCPServerRequest(LiteLLMPydanticObjectBase):
command: Optional[str] = None
args: List[str] = Field(default_factory=list)
env: Dict[str, str] = Field(default_factory=dict)
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
@model_validator(mode="before")
@classmethod
@ -1092,6 +1094,9 @@ class UpdateMCPServerRequest(LiteLLMPydanticObjectBase):
command: Optional[str] = None
args: List[str] = Field(default_factory=list)
env: Dict[str, str] = Field(default_factory=dict)
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
@model_validator(mode="before")
@classmethod
@ -1141,6 +1146,9 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
command: Optional[str] = None
args: List[str] = Field(default_factory=list)
env: Dict[str, str] = Field(default_factory=dict)
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
class MakeMCPServersPublicRequest(LiteLLMPydanticObjectBase):
@ -1160,6 +1168,9 @@ class NewSkillRequest(LiteLLMPydanticObjectBase):
file_name: Optional[str] = None # Original filename
file_type: Optional[str] = None # MIME type (e.g., "application/zip")
metadata: Optional[Dict[str, Any]] = None
authorization_url: Optional[str] = None
token_url: Optional[str] = None
registration_url: Optional[str] = None
class UpdateSkillRequest(LiteLLMPydanticObjectBase):
@ -1347,12 +1358,12 @@ class NewCustomerRequest(BudgetNewRequest):
blocked: bool = False # allow/disallow requests for this end-user
budget_id: Optional[str] = None # give either a budget_id or max_budget
spend: Optional[float] = None
allowed_model_region: Optional[AllowedModelRegion] = (
None # require all user requests to use models in this specific region
)
default_model: Optional[str] = (
None # if no equivalent model in allowed region - default all requests to this model
)
allowed_model_region: Optional[
AllowedModelRegion
] = None # require all user requests to use models in this specific region
default_model: Optional[
str
] = None # if no equivalent model in allowed region - default all requests to this model
@model_validator(mode="before")
@classmethod
@ -1374,12 +1385,12 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase):
blocked: bool = False # allow/disallow requests for this end-user
max_budget: Optional[float] = None
budget_id: Optional[str] = None # give either a budget_id or max_budget
allowed_model_region: Optional[AllowedModelRegion] = (
None # require all user requests to use models in this specific region
)
default_model: Optional[str] = (
None # if no equivalent model in allowed region - default all requests to this model
)
allowed_model_region: Optional[
AllowedModelRegion
] = None # require all user requests to use models in this specific region
default_model: Optional[
str
] = None # if no equivalent model in allowed region - default all requests to this model
class DeleteCustomerRequest(LiteLLMPydanticObjectBase):
@ -1464,15 +1475,15 @@ class NewTeamRequest(TeamBase):
] = None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm
model_tpm_limit: Optional[Dict[str, int]] = None
team_member_budget: Optional[float] = (
None # allow user to set a budget for all team members
)
team_member_rpm_limit: Optional[int] = (
None # allow user to set RPM limit for all team members
)
team_member_tpm_limit: Optional[int] = (
None # allow user to set TPM limit for all team members
)
team_member_budget: Optional[
float
] = None # allow user to set a budget for all team members
team_member_rpm_limit: Optional[
int
] = None # allow user to set RPM limit for all team members
team_member_tpm_limit: Optional[
int
] = None # allow user to set TPM limit for all team members
team_member_key_duration: Optional[str] = None # e.g. "1d", "1w", "1m"
allowed_vector_store_indexes: Optional[List[AllowedVectorStoreIndexItem]] = None
@ -1558,9 +1569,9 @@ class BlockKeyRequest(LiteLLMPydanticObjectBase):
class AddTeamCallback(LiteLLMPydanticObjectBase):
callback_name: str
callback_type: Optional[Literal["success", "failure", "success_and_failure"]] = (
"success_and_failure"
)
callback_type: Optional[
Literal["success", "failure", "success_and_failure"]
] = "success_and_failure"
callback_vars: Dict[str, str]
@model_validator(mode="before")
@ -1788,9 +1799,10 @@ class DynamoDBArgs(LiteLLMPydanticObjectBase):
class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase):
"""
Settings for a specific guardrail on a passthrough endpoint.
Allows field-level targeting for guardrail execution.
"""
request_fields: Optional[List[str]] = Field(
default=None,
description="JSONPath expressions for input field targeting (pre_call). Examples: 'query', 'documents[*].text', 'messages[*].content'. If not specified, guardrail runs on entire request payload.",
@ -1871,9 +1883,9 @@ class ConfigList(LiteLLMPydanticObjectBase):
stored_in_db: Optional[bool]
field_default_value: Any
premium_field: bool = False
nested_fields: Optional[List[FieldDetail]] = (
None # For nested dictionary or Pydantic fields
)
nested_fields: Optional[
List[FieldDetail]
] = None # For nested dictionary or Pydantic fields
class UserHeaderMapping(LiteLLMPydanticObjectBase):
@ -2152,6 +2164,7 @@ class UserAPIKeyAuth(
user_rpm_limit: Optional[int] = None
user_email: Optional[str] = None
request_route: Optional[str] = None
user: Optional[Any] = None # Expanded user object when expand=user is used
model_config = ConfigDict(arbitrary_types_allowed=True)
@ -2258,9 +2271,9 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase):
budget_id: Optional[str] = None
created_at: datetime
updated_at: datetime
user: Optional[Any] = (
None # You might want to replace 'Any' with a more specific type if available
)
user: Optional[
Any
] = None # You might want to replace 'Any' with a more specific type if available
litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
model_config = ConfigDict(protected_namespaces=())
@ -2702,7 +2715,7 @@ class AllCallbacks(LiteLLMPydanticObjectBase):
"TRACELOOP_API_KEY",
],
ui_callback_name="Traceloop",
)
)
class SpendLogsMetadata(TypedDict):
@ -2736,9 +2749,7 @@ class SpendLogsMetadata(TypedDict):
cold_storage_object_key: Optional[
str
] # S3/GCS object key for cold storage retrieval
litellm_overhead_time_ms: Optional[
float
] # LiteLLM overhead time in milliseconds
litellm_overhead_time_ms: Optional[float] # LiteLLM overhead time in milliseconds
cost_breakdown: Optional[
CostBreakdown
] # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
@ -3222,9 +3233,9 @@ class TeamModelDeleteRequest(BaseModel):
# Organization Member Requests
class OrganizationMemberAddRequest(OrgMemberAddRequest):
organization_id: str
max_budget_in_organization: Optional[float] = (
None # Users max budget within the organization
)
max_budget_in_organization: Optional[
float
] = None # Users max budget within the organization
class OrganizationMemberDeleteRequest(MemberDeleteRequest):
@ -3439,9 +3450,9 @@ class ProviderBudgetResponse(LiteLLMPydanticObjectBase):
Maps provider names to their budget configs.
"""
providers: Dict[str, ProviderBudgetResponseObject] = (
{}
) # Dictionary mapping provider names to their budget configurations
providers: Dict[
str, ProviderBudgetResponseObject
] = {} # Dictionary mapping provider names to their budget configurations
class ProxyStateVariables(TypedDict):
@ -3560,8 +3571,16 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
default=None,
description="If no team_id given, default permissions/spend-tracking to this team.s",
)
team_alias_jwt_field: Optional[str] = Field(
default=None,
description="The field in the JWT token that stores the team name/alias. Will be resolved to team_id via database lookup.",
)
org_id_jwt_field: Optional[str] = None
org_alias_jwt_field: Optional[str] = Field(
default=None,
description="The field in the JWT token that stores the organization name/alias. Will be resolved to org_id via database lookup.",
)
user_id_jwt_field: Optional[str] = None
user_email_jwt_field: Optional[str] = None
user_allowed_email_domain: Optional[str] = None
@ -3576,9 +3595,9 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
enforce_rbac: bool = False
roles_jwt_field: Optional[str] = None # v2 on role mappings
role_mappings: Optional[List[RoleMapping]] = None
object_id_jwt_field: Optional[str] = (
None # can be either user / team, inferred from the role mapping
)
object_id_jwt_field: Optional[
str
] = None # can be either user / team, inferred from the role mapping
scope_mappings: Optional[List[ScopeMapping]] = None
enforce_scope_based_access: bool = False
enforce_team_based_model_access: bool = False
@ -3729,13 +3748,16 @@ class DailyOrganizationSpendTransaction(BaseDailySpendTransaction):
class DailyUserSpendTransaction(BaseDailySpendTransaction):
user_id: str
class DailyEndUserSpendTransaction(BaseDailySpendTransaction):
end_user_id: str
class DailyTagSpendTransaction(BaseDailySpendTransaction):
request_id: Optional[str]
tag: str
class DailyAgentSpendTransaction(BaseDailySpendTransaction):
agent_id: str
@ -3767,8 +3789,8 @@ class LiteLLM_ManagedFileTable(LiteLLMPydanticObjectBase):
flat_model_file_ids: List[str]
created_by: Optional[str]
updated_by: Optional[str]
storage_backend: Optional[str] = None
storage_url: Optional[str] = None
storage_backend: Optional[str] = None
storage_url: Optional[str] = None
class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase):
@ -3799,4 +3821,4 @@ class LiteLLM_ManagedVectorStoresTable(LiteLLMPydanticObjectBase):
class ResponseLiteLLM_ManagedVectorStore(TypedDict, total=False):
vector_store: LiteLLM_ManagedVectorStoresTable
vector_store: LiteLLM_ManagedVectorStoresTable

189
litellm/proxy/auth/auth_checks.py
View file

@ -1372,6 +1372,195 @@ async def get_team_object(
)
@log_db_metrics
async def get_team_object_by_alias(
team_alias: str,
prisma_client: Optional[PrismaClient],
user_api_key_cache: DualCache,
parent_otel_span: Optional["Span"] = None,
proxy_logging_obj: Optional[ProxyLogging] = None,
) -> LiteLLM_TeamTableCachedObj:
"""
Look up a team by its team_alias (name) in the database.
Args:
team_alias: The team name/alias to look up
prisma_client: Database client
user_api_key_cache: Cache for storing results
parent_otel_span: Optional OpenTelemetry span
proxy_logging_obj: Optional proxy logging object
Returns:
LiteLLM_TeamTableCachedObj: The team object if found
Raises:
HTTPException: If team doesn't exist or multiple teams have the same alias
"""
if prisma_client is None:
raise Exception(
"No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys"
)
# Check cache first (keyed by alias)
cache_key = "team_alias:{}".format(team_alias)
cached_team_obj = await _get_team_object_from_cache(
key=cache_key,
proxy_logging_obj=proxy_logging_obj,
user_api_key_cache=user_api_key_cache,
parent_otel_span=parent_otel_span,
)
if cached_team_obj is not None:
return cached_team_obj
# Query database by team_alias
try:
teams = await prisma_client.db.litellm_teamtable.find_many(
where={"team_alias": team_alias}
)
if not teams:
raise HTTPException(
status_code=404,
detail={
"error": f"Team with alias '{team_alias}' doesn't exist in db. Create team via `/team/new` call."
},
)
if len(teams) > 1:
raise HTTPException(
status_code=400,
detail={
"error": f"Multiple teams found with alias '{team_alias}'. Please use team_id_jwt_field instead or ensure team aliases are unique."
},
)
team = teams[0]
team_obj = LiteLLM_TeamTableCachedObj(**team.model_dump())
# Cache the result by both alias and team_id
await user_api_key_cache.async_set_cache(
key=cache_key,
value=team_obj,
ttl=DEFAULT_IN_MEMORY_TTL,
)
# Also cache by team_id for consistency
team_id_cache_key = "team_id:{}".format(team_obj.team_id)
await user_api_key_cache.async_set_cache(
key=team_id_cache_key,
value=team_obj,
ttl=DEFAULT_IN_MEMORY_TTL,
)
return team_obj
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception(
"Error looking up team by alias: %s", team_alias
)
raise HTTPException(
status_code=500,
detail={
"error": f"Error looking up team by alias '{team_alias}': {str(e)}"
},
)
@log_db_metrics
async def get_org_object_by_alias(
org_alias: str,
prisma_client: Optional[PrismaClient],
user_api_key_cache: DualCache,
parent_otel_span: Optional["Span"] = None,
proxy_logging_obj: Optional[ProxyLogging] = None,
) -> Optional[LiteLLM_OrganizationTable]:
"""
Look up an organization by its organization_alias in the database.
Args:
org_alias: The organization name/alias to look up
prisma_client: Database client
user_api_key_cache: Cache for storing results
parent_otel_span: Optional OpenTelemetry span
proxy_logging_obj: Optional proxy logging object
Returns:
LiteLLM_OrganizationTable if found, None otherwise
Raises:
HTTPException: If organization not found or multiple orgs have the same alias
"""
if prisma_client is None:
raise Exception(
"No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys"
)
# Check cache first (keyed by alias)
cache_key = "org_alias:{}".format(org_alias)
cached_org_obj = await user_api_key_cache.async_get_cache(key=cache_key)
if cached_org_obj is not None:
if isinstance(cached_org_obj, dict):
return LiteLLM_OrganizationTable(**cached_org_obj)
elif isinstance(cached_org_obj, LiteLLM_OrganizationTable):
return cached_org_obj
# Query database by organization_alias
try:
orgs = await prisma_client.db.litellm_organizationtable.find_many(
where={"organization_alias": org_alias}
)
if not orgs:
raise HTTPException(
status_code=404,
detail={
"error": f"Organization with alias '{org_alias}' doesn't exist in db. Create organization via `/organization/new` call."
},
)
if len(orgs) > 1:
raise HTTPException(
status_code=400,
detail={
"error": f"Multiple organizations found with alias '{org_alias}'. Please use org_id_jwt_field instead or ensure organization aliases are unique."
},
)
org = orgs[0]
org_obj = LiteLLM_OrganizationTable(**org.model_dump())
# Cache the result
await user_api_key_cache.async_set_cache(
key=cache_key,
value=org_obj.model_dump(),
ttl=DEFAULT_IN_MEMORY_TTL,
)
# Also cache by org_id for consistency
await user_api_key_cache.async_set_cache(
key="org_id:{}".format(org_obj.organization_id),
value=org_obj.model_dump(),
ttl=DEFAULT_IN_MEMORY_TTL,
)
return org_obj
except HTTPException:
raise
except Exception as e:
verbose_proxy_logger.exception(
"Error looking up organization by alias: %s", org_alias
)
raise HTTPException(
status_code=500,
detail={
"error": f"Error looking up organization by alias '{org_alias}': {str(e)}"
},
)
class ExperimentalUIJWTToken:
@staticmethod
def get_experimental_ui_login_jwt_auth_token(user_info: LiteLLM_UserTable) -> str:

139
litellm/proxy/auth/handle_jwt.py
View file

@ -48,10 +48,12 @@ from .auth_checks import (
get_actual_routes,
get_end_user_object,
get_org_object,
get_org_object_by_alias,
get_role_based_models,
get_role_based_routes,
get_team_membership,
get_team_object,
get_team_object_by_alias,
get_user_object,
)
@ -194,10 +196,13 @@ class JWTHandler:
def is_required_team_id(self) -> bool:
"""
Returns:
- True: if 'team_id_jwt_field' is set
- False: if not
- True: if 'team_id_jwt_field' or 'team_alias_jwt_field' is set
- False: if neither is set
"""
if self.litellm_jwtauth.team_id_jwt_field is None:
if (
self.litellm_jwtauth.team_id_jwt_field is None
and self.litellm_jwtauth.team_alias_jwt_field is None
):
return False
return True
@ -240,6 +245,31 @@ class JWTHandler:
team_id = default_value
return team_id
def get_team_alias(self, token: dict, default_value: Optional[str]) -> Optional[str]:
"""
Extract team name/alias from JWT token using the configured team_alias_jwt_field.
Args:
token: The decoded JWT token dictionary
default_value: Default value to return if field not found
Returns:
The team alias from the token, or default_value if not found
"""
try:
if self.litellm_jwtauth.team_alias_jwt_field is not None:
team_alias = get_nested_value(
data=token,
key_path=self.litellm_jwtauth.team_alias_jwt_field,
default=default_value,
)
return team_alias
else:
team_alias = None
except KeyError:
team_alias = default_value
return team_alias
def is_upsert_user_id(self, valid_user_email: Optional[bool] = None) -> bool:
"""
Returns:
@ -383,6 +413,31 @@ class JWTHandler:
org_id = default_value
return org_id
def get_org_alias(self, token: dict, default_value: Optional[str]) -> Optional[str]:
"""
Extract organization name/alias from JWT token using the configured org_alias_jwt_field.
Args:
token: The decoded JWT token dictionary
default_value: Default value to return if field not found
Returns:
The organization alias from the token, or default_value if not found
"""
try:
if self.litellm_jwtauth.org_alias_jwt_field is not None:
org_alias = get_nested_value(
data=token,
key_path=self.litellm_jwtauth.org_alias_jwt_field,
default=default_value,
)
return org_alias
else:
org_alias = None
except KeyError:
org_alias = default_value
return org_alias
def get_scopes(self, token: dict) -> List[str]:
try:
if isinstance(token["scope"], str):
@ -813,18 +868,14 @@ class JWTAuthManager:
parent_otel_span: Optional[Span],
proxy_logging_obj: ProxyLogging,
) -> Tuple[Optional[str], Optional[LiteLLM_TeamTable]]:
"""Find and validate specific team ID"""
"""Find and validate specific team ID from team_id_jwt_field or team_alias_jwt_field"""
individual_team_id = jwt_handler.get_team_id(
token=jwt_valid_token, default_value=None
)
if not individual_team_id and jwt_handler.is_required_team_id() is True:
raise Exception(
f"No team id found in token. Checked team_id field '{jwt_handler.litellm_jwtauth.team_id_jwt_field}'"
)
## VALIDATE TEAM OBJECT ###
team_object: Optional[LiteLLM_TeamTable] = None
# First try to get team by team_id
if individual_team_id:
team_object = await get_team_object(
team_id=individual_team_id,
@ -834,6 +885,37 @@ class JWTAuthManager:
proxy_logging_obj=proxy_logging_obj,
team_id_upsert=jwt_handler.litellm_jwtauth.team_id_upsert,
)
return individual_team_id, team_object
# If no team_id found, try to resolve via team_alias_jwt_field
team_alias = jwt_handler.get_team_alias(
token=jwt_valid_token, default_value=None
)
if team_alias:
verbose_proxy_logger.info(
f"JWT Auth: Resolving team by alias: '{team_alias}'"
)
team_object = await get_team_object_by_alias(
team_alias=team_alias,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
if team_object:
individual_team_id = team_object.team_id
verbose_proxy_logger.info(
f"JWT Auth: Resolved team_alias='{team_alias}' to team_id='{individual_team_id}'"
)
return individual_team_id, team_object
# Check if team is required but not found
if jwt_handler.is_required_team_id() is True:
team_id_field = jwt_handler.litellm_jwtauth.team_id_jwt_field
team_alias_field = jwt_handler.litellm_jwtauth.team_alias_jwt_field
raise Exception(
f"No team found in token. Checked team_id field '{team_id_field}' and team_alias field '{team_alias_field}'"
)
return individual_team_id, team_object
@ -942,13 +1024,16 @@ class JWTAuthManager:
parent_otel_span: Optional[Span],
proxy_logging_obj: ProxyLogging,
route: str,
org_alias: Optional[str] = None,
) -> Tuple[
Optional[LiteLLM_UserTable],
Optional[LiteLLM_OrganizationTable],
Optional[LiteLLM_EndUserTable],
Optional[LiteLLM_EndUserTable],
Optional[LiteLLM_TeamMembership],
]:
"""Get user, org, and end user objects"""
"""Get user, org, and end user objects. Also resolves org aliases to IDs if configured."""
# Get org object - first try by ID, then by alias
org_object: Optional[LiteLLM_OrganizationTable] = None
if org_id:
org_object = (
@ -962,6 +1047,21 @@ class JWTAuthManager:
if org_id
else None
)
elif org_alias:
verbose_proxy_logger.info(
f"JWT Auth: Resolving org by alias: '{org_alias}'"
)
org_object = await get_org_object_by_alias(
org_alias=org_alias,
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
if org_object:
verbose_proxy_logger.info(
f"JWT Auth: Resolved org_alias='{org_alias}' to org_id='{org_object.organization_id}'"
)
user_object: Optional[LiteLLM_UserTable] = None
if user_id:
@ -1304,6 +1404,8 @@ class JWTAuthManager:
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
)
# Extract alias fields for resolution (if configured)
org_alias = jwt_handler.get_org_alias(token=jwt_valid_token, default_value=None)
# Get other objects
user_object, org_object, end_user_object, team_membership_object = (
@ -1320,9 +1422,13 @@ class JWTAuthManager:
parent_otel_span=parent_otel_span,
proxy_logging_obj=proxy_logging_obj,
route=route,
org_alias=org_alias,
)
)
# Derive org_id from org_object if resolved by alias
resolved_org_id = org_object.organization_id if org_object else org_id
await JWTAuthManager.sync_user_role_and_teams(
jwt_handler=jwt_handler,
jwt_valid_token=jwt_valid_token,
@ -1345,10 +1451,9 @@ class JWTAuthManager:
)
# check if user is proxy admin
if user_object and user_object.user_role == LitellmUserRoles.PROXY_ADMIN:
is_proxy_admin = True
else:
is_proxy_admin = False
is_proxy_admin = bool(
user_object and user_object.user_role == LitellmUserRoles.PROXY_ADMIN
)
return JWTAuthBuilderResult(
is_proxy_admin=is_proxy_admin,
@ -1356,7 +1461,7 @@ class JWTAuthManager:
team_object=team_object,
user_id=user_id,
user_object=user_object,
org_id=org_id,
org_id=resolved_org_id, # Use resolved org_id (from alias lookup if applicable)
org_object=org_object,
end_user_id=end_user_id,
end_user_object=end_user_object,

64
litellm/proxy/auth/login_utils.py
View file

@ -34,6 +34,59 @@ from litellm.secret_managers.main import get_secret_bool
from litellm.types.proxy.ui_sso import ReturnedUITokenObject
async def expire_previous_ui_session_tokens(
user_id: str, prisma_client: Optional[PrismaClient]
) -> None:
"""
Expire (block) all other valid UI session tokens for a user.
This prevents accumulation of multiple valid UI session tokens that
are supposed to be short-lived test keys. Only affects keys with
team_id = "litellm-dashboard" and that haven't expired yet.
Args:
user_id: The user ID whose previous UI session tokens should be expired
prisma_client: Database client for performing the update
"""
if prisma_client is None:
return
try:
from datetime import datetime, timezone
current_time = datetime.now(timezone.utc)
# Find all unblocked AND non-expired UI session tokens for this user
ui_session_tokens = await prisma_client.db.litellm_verificationtoken.find_many(
where={
"user_id": user_id,
"team_id": "litellm-dashboard",
"OR": [
{"blocked": None}, # Tokens that have never been blocked (null)
{"blocked": False}, # Tokens explicitly set to not blocked
],
"expires": {"gt": current_time}, # Only get tokens that haven't expired
}
)
if not ui_session_tokens:
return
# Block all the found tokens
tokens_to_block = [token.token for token in ui_session_tokens if token.token]
if tokens_to_block:
await prisma_client.db.litellm_verificationtoken.update_many(
where={"token": {"in": tokens_to_block}},
data={"blocked": True}
)
except Exception:
# Silently fail - don't block login if cleanup fails
# This is a best-effort operation
pass
def get_ui_credentials(master_key: Optional[str]) -> tuple[str, str]:
"""
Get UI username and password from environment variables or master key.
@ -85,7 +138,7 @@ class LoginResult:
self.login_method = login_method
async def authenticate_user(
async def authenticate_user( # noqa: PLR0915
username: str,
password: str,
master_key: Optional[str],
@ -174,6 +227,10 @@ async def authenticate_user(
)
if os.getenv("DATABASE_URL") is not None:
# Expire any previous UI session tokens for this user
await expire_previous_ui_session_tokens(
user_id=key_user_id, prisma_client=prisma_client
)
response = await generate_key_helper_fn(
request_type="key",
**{
@ -260,6 +317,11 @@ async def authenticate_user(
hash_password, _password
):
if os.getenv("DATABASE_URL") is not None:
# Expire any previous UI session tokens for this user
await expire_previous_ui_session_tokens(
user_id=user_id, prisma_client=prisma_client
)
response = await generate_key_helper_fn(
request_type="key",
**{ # type: ignore

3
litellm/proxy/auth/user_api_key_auth.py
View file

@ -551,6 +551,9 @@ async def _user_api_key_auth_builder( # noqa: PLR0915
valid_token = UserAPIKeyAuth(
api_key=None,
team_id=team_id,
team_alias=(
team_object.team_alias if team_object is not None else None
),
team_tpm_limit=(
team_object.tpm_limit if team_object is not None else None
),

24
litellm/proxy/management_endpoints/budget_management_endpoints.py
View file

@ -55,6 +55,18 @@ async def new_budget(
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
# Validate budget values are not negative
if budget_obj.max_budget is not None and budget_obj.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {budget_obj.max_budget}"}
)
if budget_obj.soft_budget is not None and budget_obj.soft_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"soft_budget cannot be negative. Received: {budget_obj.soft_budget}"}
)
# if no budget_reset_at date is set, but a budget_duration is given, then set budget_reset_at initially to the first completed duration interval in future
if budget_obj.budget_reset_at is None and budget_obj.budget_duration is not None:
budget_obj.budget_reset_at = datetime.utcnow() + timedelta(
@ -107,6 +119,18 @@ async def update_budget(
if budget_obj.budget_id is None:
raise HTTPException(status_code=400, detail={"error": "budget_id is required"})
# Validate budget values are not negative
if budget_obj.max_budget is not None and budget_obj.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {budget_obj.max_budget}"}
)
if budget_obj.soft_budget is not None and budget_obj.soft_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"soft_budget cannot be negative. Received: {budget_obj.soft_budget}"}
)
response = await prisma_client.db.litellm_budgettable.update(
where={"budget_id": budget_obj.budget_id},
data={

137
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -1069,6 +1069,18 @@ async def generate_key_fn(
verbose_proxy_logger.debug("entered /key/generate")
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
if data.soft_budget is not None and data.soft_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"soft_budget cannot be negative. Received: {data.soft_budget}"}
)
if user_custom_key_generate is not None:
if asyncio.iscoroutinefunction(user_custom_key_generate):
result = await user_custom_key_generate(data) # type: ignore
@ -1502,6 +1514,13 @@ async def update_key_fn(
)
try:
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
data_json: dict = data.model_dump(exclude_unset=True, exclude_none=True)
key = data_json.pop("key")
@ -3020,10 +3039,14 @@ async def list_keys(
description="Column to sort by (e.g. 'user_id', 'created_at', 'spend')",
),
sort_order: str = Query(default="desc", description="Sort order ('asc' or 'desc')"),
expand: Optional[List[str]] = Query(None, description="Expand related objects (e.g. 'user')"),
) -> KeyListResponseObject:
"""
List all keys for a given user / team / organization.
Parameters:
expand: Optional[List[str]] - Expand related objects (e.g. 'user' to include user information)
Returns:
{
"keys": List[str] or List[UserAPIKeyAuth],
@ -3031,6 +3054,9 @@ async def list_keys(
"current_page": int,
"total_pages": int,
}
When expand includes "user", each key object will include a "user" field with the associated user object.
Note: When expand=user is specified, full key objects are returned regardless of the return_full_object parameter.
"""
try:
from litellm.proxy.proxy_server import prisma_client
@ -3080,6 +3106,7 @@ async def list_keys(
include_created_by_keys=include_created_by_keys,
sort_by=sort_by,
sort_order=sort_order,
expand=expand,
)
verbose_proxy_logger.debug("Successfully prepared response")
@ -3215,45 +3242,17 @@ def _validate_sort_params(
return order_by
async def _list_key_helper(
prisma_client: PrismaClient,
page: int,
size: int,
def _build_key_filter_conditions(
user_id: Optional[str],
team_id: Optional[str],
organization_id: Optional[str],
key_alias: Optional[str],
key_hash: Optional[str],
exclude_team_id: Optional[str] = None,
return_full_object: bool = False,
admin_team_ids: Optional[
List[str]
] = None, # New parameter for teams where user is admin
include_created_by_keys: bool = False,
sort_by: Optional[str] = None,
sort_order: str = "desc",
) -> KeyListResponseObject:
"""
Helper function to list keys
Args:
page: int
size: int
user_id: Optional[str]
team_id: Optional[str]
key_alias: Optional[str]
exclude_team_id: Optional[str] # exclude a specific team_id
return_full_object: bool # when true, will return UserAPIKeyAuth objects instead of just the token
admin_team_ids: Optional[List[str]] # list of team IDs where the user is an admin
Returns:
KeyListResponseObject
{
"keys": List[str] or List[UserAPIKeyAuth], # Updated to reflect possible return types
"total_count": int,
"current_page": int,
"total_pages": int,
}
"""
exclude_team_id: Optional[str],
admin_team_ids: Optional[List[str]],
include_created_by_keys: bool,
) -> Dict[str, Union[str, Dict[str, Any], List[Dict[str, Any]]]]:
"""Build filter conditions for key listing."""
# Prepare filter conditions
where: Dict[str, Union[str, Dict[str, Any], List[Dict[str, Any]]]] = {}
where.update(_get_condition_to_filter_out_ui_session_tokens())
@ -3294,6 +3293,59 @@ async def _list_key_helper(
where.update(or_conditions[0])
verbose_proxy_logger.debug(f"Filter conditions: {where}")
return where
async def _list_key_helper(
prisma_client: PrismaClient,
page: int,
size: int,
user_id: Optional[str],
team_id: Optional[str],
organization_id: Optional[str],
key_alias: Optional[str],
key_hash: Optional[str],
exclude_team_id: Optional[str] = None,
return_full_object: bool = False,
admin_team_ids: Optional[
List[str]
] = None, # New parameter for teams where user is admin
include_created_by_keys: bool = False,
sort_by: Optional[str] = None,
sort_order: str = "desc",
expand: Optional[List[str]] = None,
) -> KeyListResponseObject:
"""
Helper function to list keys
Args:
page: int
size: int
user_id: Optional[str]
team_id: Optional[str]
key_alias: Optional[str]
exclude_team_id: Optional[str] # exclude a specific team_id
return_full_object: bool # when true, will return UserAPIKeyAuth objects instead of just the token
admin_team_ids: Optional[List[str]] # list of team IDs where the user is an admin
Returns:
KeyListResponseObject
{
"keys": List[str] or List[UserAPIKeyAuth], # Updated to reflect possible return types
"total_count": int,
"current_page": int,
"total_pages": int,
}
"""
where = _build_key_filter_conditions(
user_id=user_id,
team_id=team_id,
organization_id=organization_id,
key_alias=key_alias,
key_hash=key_hash,
exclude_team_id=exclude_team_id,
admin_team_ids=admin_team_ids,
include_created_by_keys=include_created_by_keys,
)
# Calculate skip for pagination
skip = (page - 1) * size
@ -3334,13 +3386,28 @@ async def _list_key_helper(
# Calculate total pages
total_pages = -(-total_count // size) # Ceiling division
# Fetch user information if expand includes "user"
user_map = {}
if expand and "user" in expand:
user_ids = [key.user_id for key in keys if key.user_id]
if user_ids:
users = await prisma_client.db.litellm_usertable.find_many(
where={"user_id": {"in": list(set(user_ids))}} # Remove duplicates
)
user_map = {user.user_id: user for user in users}
# Prepare response
key_list: List[Union[str, UserAPIKeyAuth]] = []
for key in keys:
key_dict = key.dict()
# Attach object_permission if object_permission_id is set
key_dict = await attach_object_permission_to_dict(key_dict, prisma_client)
if return_full_object is True:
# Include user information if expand includes "user"
if expand and "user" in expand and key.user_id and key.user_id in user_map:
key_dict["user"] = user_map[key.user_id].dict()
if return_full_object is True or (expand and "user" in expand):
key_list.append(UserAPIKeyAuth(**key_dict)) # Return full key object
else:
_token = key_dict.get("token")

187
litellm/proxy/management_endpoints/mcp_management_endpoints.py
View file

@ -16,7 +16,7 @@ Endpoints here:
import importlib
from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import Any, Dict, Iterable, List, Optional
from typing import Any, Dict, Iterable, List, Literal, Optional
from fastapi import (
APIRouter,
@ -24,6 +24,7 @@ from fastapi import (
Form,
Header,
HTTPException,
Query,
Request,
Response,
status,
@ -208,6 +209,9 @@ if MCP_AVAILABLE:
command=payload.command,
args=payload.args,
env=payload.env,
authorization_url=payload.authorization_url,
token_url=payload.token_url,
registration_url=payload.registration_url,
)
def get_prisma_client_or_throw(message: str):
@ -296,117 +300,6 @@ if MCP_AVAILABLE:
access_groups_list = sorted(list(access_groups))
return {"access_groups": access_groups_list}
@router.get(
"/server/{server_id}/health",
description="Perform health check on a specific MCP server",
dependencies=[Depends(user_api_key_auth)],
)
async def health_check_mcp_server(
server_id: str,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Perform a health check on the MCP server specified by the `server_id`
Parameters:
- server_id: str - Required. The unique identifier of the mcp server to health check.
```
curl --location 'http://localhost:4000/v1/mcp/server/{server_id}/health' \
--header 'Authorization: Bearer your_api_key_here'
```
"""
# Check if server exists and user has access
prisma_client = get_prisma_client_or_throw(
"Database not connected. Connect a database to your proxy"
)
# check to see if server exists for all users
mcp_server = await get_mcp_server(prisma_client, server_id)
if mcp_server is None:
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail={"error": f"MCP Server with id {server_id} not found"},
)
# Implement authz restriction from requested user
if not _user_has_admin_view(user_api_key_dict):
# Perform authz check to filter the mcp servers user has access to
mcp_server_records = await get_all_mcp_servers_for_user(
prisma_client, user_api_key_dict
)
exists = does_mcp_server_exist(mcp_server_records, server_id)
if not exists:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail={
"error": f"User does not have permission to access mcp server with id {server_id}. You can only access mcp servers that you have access to."
},
)
# Perform health check using server manager
try:
health_result = await global_mcp_server_manager.health_check_server(
server_id
)
return health_result
except Exception as e:
verbose_proxy_logger.exception(
f"Error performing health check on MCP server {server_id}: {str(e)}"
)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={"error": f"Error performing health check: {str(e)}"},
)
@router.get(
"/server/health",
description="Perform health check on all accessible MCP servers",
dependencies=[Depends(user_api_key_auth)],
)
async def health_check_all_mcp_servers(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Perform health checks on all MCP servers accessible to the user
```
curl --location 'http://localhost:4000/v1/mcp/server/health' \
--header 'Authorization: Bearer your_api_key_here'
```
"""
# Use server manager to get health checks for allowed servers
try:
all_health_results = (
await global_mcp_server_manager.health_check_allowed_servers(
user_api_key_auth=user_api_key_dict
)
)
return {
"total_servers": len(all_health_results),
"healthy_count": len(
[r for r in all_health_results.values() if r["status"] == "healthy"]
),
"unhealthy_count": len(
[
r
for r in all_health_results.values()
if r["status"] == "unhealthy"
]
),
"unknown_count": len(
[r for r in all_health_results.values() if r["status"] == "unknown"]
),
"servers": all_health_results,
}
except Exception as e:
verbose_proxy_logger.exception(
f"Error performing health checks on MCP servers: {str(e)}"
)
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail={"error": f"Error performing health checks: {str(e)}"},
)
## FastAPI Routes
@router.get(
"/server",
@ -429,7 +322,7 @@ if MCP_AVAILABLE:
aggregated_servers: Dict[str, LiteLLM_MCPServerTable] = {}
for auth_context in auth_contexts:
servers = await global_mcp_server_manager.get_all_mcp_servers_with_health_and_teams(
servers = await global_mcp_server_manager.get_all_allowed_mcp_servers(
user_api_key_auth=auth_context
)
for server in servers:
@ -447,6 +340,56 @@ if MCP_AVAILABLE:
server.mcp_info["is_public"] = True
return redacted_mcp_servers
@router.get(
"/server/health",
description="Health check for MCP servers",
dependencies=[Depends(user_api_key_auth)],
)
async def health_check_servers(
server_ids: Optional[List[str]] = Query(
None,
description="Server IDs to check. If not provided, checks all accessible servers.",
),
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Perform health checks on one or more MCP servers.
Parameters:
- server_ids: Optional list of server IDs. If not provided, checks all accessible servers.
Returns:
- Health check results for requested servers
```
# Check all accessible servers
curl --location 'http://localhost:4000/v1/mcp/server/health' \
--header 'Authorization: Bearer your_api_key_here'
# Check specific servers
curl --location 'http://localhost:4000/v1/mcp/server/health?server_ids=server-1&server_ids=server-2' \
--header 'Authorization: Bearer your_api_key_here'
```
"""
auth_contexts = await build_effective_auth_contexts(user_api_key_dict)
server_status_map: Dict[
str, Optional[Literal["healthy", "unhealthy", "unknown"]]
] = {}
for auth_context in auth_contexts:
servers = await global_mcp_server_manager.get_all_mcp_servers_with_health_and_teams(
user_api_key_auth=auth_context,
server_ids=server_ids,
)
for server in servers:
if server.server_id not in server_status_map:
server_status_map[server.server_id] = server.status
return [
{"server_id": server_id, "status": status}
for server_id, status in server_status_map.items()
]
@router.get(
"/server/{server_id}",
description="Returns the mcp server info",
@ -484,15 +427,11 @@ if MCP_AVAILABLE:
server_id
)
# Update the server object with health check results
mcp_server.status = health_result.get("status", "unknown")
mcp_server.last_health_check = (
datetime.fromisoformat(
health_result.get("last_health_check", datetime.now().isoformat())
)
if health_result.get("last_health_check")
else None
mcp_server.status = (
health_result.status if health_result.status else "unknown"
)
mcp_server.health_check_error = health_result.get("error")
mcp_server.last_health_check = health_result.last_health_check
mcp_server.health_check_error = health_result.health_check_error
except Exception as e:
verbose_proxy_logger.debug(
f"Error performing health check on server {server_id}: {e}"
@ -512,7 +451,7 @@ if MCP_AVAILABLE:
exists = does_mcp_server_exist(mcp_server_records, server_id)
if exists:
await global_mcp_server_manager.add_update_server(mcp_server)
await global_mcp_server_manager.add_server(mcp_server)
return _redact_mcp_credentials(mcp_server)
else:
raise HTTPException(
@ -586,7 +525,7 @@ if MCP_AVAILABLE:
payload,
touched_by=user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
)
await global_mcp_server_manager.add_update_server(new_mcp_server)
await global_mcp_server_manager.add_server(new_mcp_server)
# Ensure registry is up to date by reloading from database
await global_mcp_server_manager.reload_servers_from_database()
@ -867,7 +806,7 @@ if MCP_AVAILABLE:
"error": f"MCP Server not found, passed server_id={payload.server_id}"
},
)
await global_mcp_server_manager.add_update_server(mcp_server_record_updated)
await global_mcp_server_manager.update_server(mcp_server_record_updated)
# Ensure registry is up to date by reloading from database
await global_mcp_server_manager.reload_servers_from_database()

34
litellm/proxy/management_endpoints/organization_endpoints.py
View file

@ -24,8 +24,11 @@ from litellm.proxy.management_endpoints.budget_management_endpoints import (
new_budget,
update_budget,
)
from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity
from litellm.proxy.management_endpoints.common_utils import (
_set_object_metadata_field,
_user_has_admin_view,
)
from litellm.proxy.management_helpers.object_permission_utils import (
handle_update_object_permission_common,
)
@ -34,11 +37,10 @@ from litellm.proxy.management_helpers.utils import (
management_endpoint_wrapper,
)
from litellm.proxy.utils import PrismaClient
from litellm.utils import _update_dictionary
from litellm.types.proxy.management_endpoints.common_daily_activity import (
SpendAnalyticsPaginatedResponse,
)
from litellm.proxy.management_endpoints.common_daily_activity import get_daily_activity
from litellm.utils import _update_dictionary
router = APIRouter()
@ -168,6 +170,18 @@ async def new_organization(
status_code=500, detail={"error": CommonProxyErrors.no_llm_router.value}
)
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
if data.soft_budget is not None and data.soft_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"soft_budget cannot be negative. Received: {data.soft_budget}"}
)
user_object_correct_type: Optional[LiteLLM_UserTable] = None
if user_api_key_dict.user_id is not None:
@ -414,6 +428,18 @@ async def update_organization(
# Create validated data model
data = LiteLLM_OrganizationTableUpdate(**raw_data_with_flat_budget_fields)
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
if data.soft_budget is not None and data.soft_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"soft_budget cannot be negative. Received: {data.soft_budget}"}
)
if data.updated_by is None:
data.updated_by = user_api_key_dict.user_id

26
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -732,6 +732,18 @@ async def new_team( # noqa: PLR0915
if prisma_client is None:
raise HTTPException(status_code=500, detail={"error": "No db connected"})
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
if data.team_member_budget is not None and data.team_member_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"team_member_budget cannot be negative. Received: {data.team_member_budget}"}
)
# Check if license is over limit
total_teams = await prisma_client.db.litellm_teamtable.count()
if total_teams and _license_check.is_team_count_over_limit(
@ -1169,7 +1181,7 @@ def validate_team_org_change(
"/team/update", tags=["team management"], dependencies=[Depends(user_api_key_auth)]
)
@management_endpoint_wrapper
async def update_team(
async def update_team( # noqa: PLR0915
data: UpdateTeamRequest,
http_request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@ -1254,6 +1266,18 @@ async def update_team(
raise HTTPException(status_code=400, detail={"error": "No team id passed in"})
verbose_proxy_logger.debug("/team/update - %s", data)
# Validate budget values are not negative
if data.max_budget is not None and data.max_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"max_budget cannot be negative. Received: {data.max_budget}"}
)
if data.team_member_budget is not None and data.team_member_budget < 0:
raise HTTPException(
status_code=400,
detail={"error": f"team_member_budget cannot be negative. Received: {data.team_member_budget}"}
)
existing_team_row = await prisma_client.db.litellm_teamtable.find_unique(
where={"team_id": data.team_id}
)

27
litellm/proxy/proxy_server.py
View file

@ -4620,7 +4620,7 @@ class ProxyStartupEvent:
verbose_proxy_logger.info("Batch cost check job scheduled successfully")
except Exception as e:
verbose_proxy_logger.error(f"Failed to setup batch cost checking: {e}")
verbose_proxy_logger.debug(f"Failed to setup batch cost checking: {e}")
verbose_proxy_logger.debug(
"Checking batch cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..."
)
@ -4653,9 +4653,7 @@ class ProxyStartupEvent:
)
except Exception as e:
verbose_proxy_logger.error(
f"Failed to setup responses cost checking: {e}"
)
verbose_proxy_logger.debug(f"Failed to setup responses cost checking: {e}")
verbose_proxy_logger.debug(
"Checking responses cost for LiteLLM Managed Files is an Enterprise Feature. Skipping..."
)
@ -7319,13 +7317,9 @@ async def model_info_v2(
"""
global llm_model_list, general_settings, user_config_file_path, proxy_config, llm_router
if llm_router is None:
raise HTTPException(
status_code=500,
detail={
"error": f"No model list passed, models router={llm_router}. You can add a model through the config.yaml or on the LiteLLM Admin UI."
},
)
# Return empty data array when no models are configured (graceful handling for fresh installs)
if llm_router is None or not llm_router.model_list:
return {"data": []}
if prisma_client is None:
raise HTTPException(
@ -8225,14 +8219,9 @@ async def model_group_info(
"""
global llm_model_list, general_settings, user_config_file_path, proxy_config, llm_router
if llm_model_list is None:
raise HTTPException(
status_code=500, detail={"error": "LLM Model List not loaded in"}
)
if llm_router is None:
raise HTTPException(
status_code=500, detail={"error": "LLM Router is not loaded in"}
)
# Return empty data array when no models are configured (graceful handling for fresh installs)
if llm_model_list is None or llm_router is None or not llm_model_list:
return {"data": []}
from litellm.proxy.utils import get_available_models_for_user

74
litellm/proxy/public_endpoints/provider_create_fields.json
View file

@ -1680,6 +1680,34 @@
],
"default_model_placeholder": "gpt-3.5-turbo"
},
{
"provider": "MINIMAX",
"provider_display_name": "MiniMax",
"litellm_provider": "minimax",
"credential_fields": [
{
"key": "api_key",
"label": "API Key",
"placeholder": "your-minimax-api-key",
"tooltip": "MiniMax API Key from https://platform.minimaxi.com/",
"required": true,
"field_type": "password",
"options": null,
"default_value": null
},
{
"key": "api_base",
"label": "API Base URL",
"placeholder": "https://api.minimax.io/v1",
"tooltip": "International: https://api.minimax.io/v1, China: https://api.minimaxi.com/v1",
"required": false,
"field_type": "text",
"options": null,
"default_value": "https://api.minimax.io/v1"
}
],
"default_model_placeholder": "minimax/MiniMax-M2"
},
{
"provider": "MOONSHOT",
"provider_display_name": "Moonshot",
@ -2865,7 +2893,7 @@
"key": "api_base",
"label": "API Base",
"placeholder": null,
"tooltip": null,
"tooltip": "Base URL of your WatsonX instance",
"required": false,
"field_type": "text",
"options": null,
@ -2875,14 +2903,54 @@
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"tooltip": "IBM Cloud API key. Required if not using Token or Zen API Key",
"required": false,
"field_type": "password",
"options": null,
"default_value": null
},
{
"key": "token",
"label": "IAM Token",
"placeholder": null,
"tooltip": "Pre-generated IAM bearer token. Use instead of API Key if you manage tokens externally",
"required": false,
"field_type": "password",
"options": null,
"default_value": null
},
{
"key": "zen_api_key",
"label": "Zen API Key",
"placeholder": null,
"tooltip": "Zen API Key for Cloud Pak for Data deployments. Use instead of API Key for on-premises",
"required": false,
"field_type": "password",
"options": null,
"default_value": null
},
{
"key": "project_id",
"label": "Project ID",
"placeholder": null,
"tooltip": "Optional: Your Watsonx.ai Project ID",
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "space_id",
"label": "Deployment Space ID",
"placeholder": null,
"tooltip": "Optional: Watsonx.ai Deployment Space ID",
"required": false,
"field_type": "text",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "gpt-3.5-turbo"
"default_model_placeholder": "watsonx/ibm/granite-3-3-8b-instruct"
},
{
"provider": "WATSONX_TEXT",

3
litellm/proxy/schema.prisma
View file

@ -208,6 +208,9 @@ model LiteLLM_MCPServerTable {
command String?
args String[] @default([])
env Json? @default("{}")
authorization_url String?
token_url String?
registration_url String?
}
// Generate Tokens for Proxy

18
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -11,7 +11,10 @@ from pydantic import BaseModel
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB, REDACTED_BY_LITELM_STRING
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.litellm_core_utils.core_helpers import (
get_litellm_metadata_from_kwargs,
reconstruct_model_name,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
from litellm.proxy.utils import PrismaClient, hash_token
@ -100,9 +103,9 @@ def _get_spend_logs_metadata(
clean_metadata["applied_guardrails"] = applied_guardrails
clean_metadata["batch_models"] = batch_models
clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata
clean_metadata["vector_store_request_metadata"] = (
_get_vector_store_request_for_spend_logs_payload(vector_store_request_metadata)
)
clean_metadata[
"vector_store_request_metadata"
] = _get_vector_store_request_for_spend_logs_payload(vector_store_request_metadata)
clean_metadata["guardrail_information"] = guardrail_information
clean_metadata["usage_object"] = usage_object
clean_metadata["model_map_information"] = model_map_information
@ -393,6 +396,9 @@ def get_logging_payload( # noqa: PLR0915
# Extract agent_id for A2A requests (set directly on model_call_details)
agent_id: Optional[str] = kwargs.get("agent_id")
custom_llm_provider = kwargs.get("custom_llm_provider")
raw_model = cast(str, kwargs.get("model") or "")
model_name = reconstruct_model_name(raw_model, custom_llm_provider, metadata or {})
try:
payload: SpendLogsPayload = SpendLogsPayload(
@ -403,7 +409,7 @@ def get_logging_payload( # noqa: PLR0915
startTime=_ensure_datetime_utc(start_time),
endTime=_ensure_datetime_utc(end_time),
completionStartTime=_ensure_datetime_utc(completion_start_time),
model=kwargs.get("model", "") or "",
model=model_name,
user=metadata.get("user_api_key_user_id", "") or "",
team_id=metadata.get("user_api_key_team_id", "") or "",
organization_id=metadata.get("user_api_key_org_id") or "",
@ -449,7 +455,7 @@ def get_logging_payload( # noqa: PLR0915
# Explicitly clear large intermediate objects to reduce memory pressure
del response_obj_dict, usage, clean_metadata, additional_usage_values
return payload
except Exception as e:
verbose_proxy_logger.exception(

2
litellm/realtime_api/main.py
View file

@ -3,7 +3,7 @@
from typing import Any, Optional, cast
import litellm
from litellm import get_llm_provider
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler

10
litellm/responses/mcp/litellm_proxy_mcp_handler.py
View file

@ -205,7 +205,15 @@ class LiteLLM_Proxy_MCP_Handler:
else:
tool_name = getattr(mcp_tool, "name", None)
if tool_name and tool_name in allowed_tool_names:
if not tool_name:
continue
if tool_name in allowed_tool_names:
filtered_tools.append(mcp_tool)
continue
unprefixed_name, _ = split_server_prefix_from_name(tool_name)
if unprefixed_name in allowed_tool_names:
filtered_tools.append(mcp_tool)
return filtered_tools

26
litellm/responses/utils.py
View file

@ -26,7 +26,7 @@ from litellm.types.llms.openai import (
from litellm.types.responses.main import DecodedResponseId
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetails,
PromptTokensDetailsWrapper,
SpecialEnums,
Usage,
)
@ -431,7 +431,12 @@ class ResponseAPILoggingUtils:
def _transform_response_api_usage_to_chat_usage(
usage_input: Optional[Union[dict, ResponseAPIUsage]],
) -> Usage:
"""Tranforms the ResponseAPIUsage object to a Usage object"""
"""
Transforms ResponseAPIUsage or ImageUsage to a Usage object.
Both have the same spec with input_tokens, output_tokens, and
input_tokens_details (text_tokens, image_tokens).
"""
if usage_input is None:
return Usage(
prompt_tokens=0,
@ -445,18 +450,19 @@ class ResponseAPILoggingUtils:
)
prompt_tokens: int = response_api_usage.input_tokens or 0
completion_tokens: int = response_api_usage.output_tokens or 0
prompt_tokens_details: Optional[PromptTokensDetails] = None
prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
if response_api_usage.input_tokens_details:
prompt_tokens_details = PromptTokensDetails(
cached_tokens=response_api_usage.input_tokens_details.cached_tokens,
audio_tokens=response_api_usage.input_tokens_details.audio_tokens,
prompt_tokens_details = PromptTokensDetailsWrapper(
cached_tokens=getattr(response_api_usage.input_tokens_details, "cached_tokens", None),
audio_tokens=getattr(response_api_usage.input_tokens_details, "audio_tokens", None),
text_tokens=getattr(response_api_usage.input_tokens_details, "text_tokens", None),
image_tokens=getattr(response_api_usage.input_tokens_details, "image_tokens", None),
)
completion_tokens_details: Optional[CompletionTokensDetailsWrapper] = None
if response_api_usage.output_tokens_details:
output_tokens_details = getattr(response_api_usage, "output_tokens_details", None)
if output_tokens_details:
completion_tokens_details = CompletionTokensDetailsWrapper(
reasoning_tokens=getattr(
response_api_usage.output_tokens_details, "reasoning_tokens", None
)
reasoning_tokens=getattr(output_tokens_details, "reasoning_tokens", None)
)
chat_usage = Usage(

2
litellm/router_utils/pattern_match_deployments.py
View file

@ -7,7 +7,7 @@ import re
from re import Match
from typing import Dict, List, Optional, Tuple
from litellm import get_llm_provider
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm._logging import verbose_router_logger

33
litellm/types/llms/openai.py
View file

@ -1197,6 +1197,39 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject):
# Define private attributes using PrivateAttr
_hidden_params: dict = PrivateAttr(default_factory=dict)
@property
def output_text(self) -> str:
"""
Convenience property that aggregates all `output_text` items from the `output` list.
If no `output_text` content blocks exist, then an empty string is returned.
This matches the OpenAI SDK's Response.output_text behavior.
"""
texts: List[str] = []
for output_item in self.output:
# Handle both dict and object access patterns
if isinstance(output_item, dict):
item_type = output_item.get("type")
content = output_item.get("content", [])
else:
item_type = getattr(output_item, "type", None)
content = getattr(output_item, "content", [])
if item_type == "message":
for content_item in content:
if isinstance(content_item, dict):
content_type = content_item.get("type")
text = content_item.get("text", "")
else:
content_type = getattr(content_item, "type", None)
text = getattr(content_item, "text", "") or ""
if content_type == "output_text":
texts.append(text)
return "".join(texts)
class ResponsesAPIStreamEvents(str, Enum):
"""

3
litellm/types/utils.py
View file

@ -142,6 +142,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
] # only for vertex ai models
input_cost_per_query: Optional[float] # only for rerank models
input_cost_per_image: Optional[float] # only for vertex ai models
input_cost_per_image_token: Optional[float] # for gpt-image-1 and similar models
input_cost_per_audio_per_second: Optional[float] # only for vertex ai models
input_cost_per_video_per_second: Optional[float] # only for vertex ai models
input_cost_per_second: Optional[float] # for OpenAI Speech models
@ -1300,7 +1301,7 @@ class CacheCreationTokenDetails(BaseModel):
class PromptTokensDetailsWrapper(
PromptTokensDetails
): # wrapper for older openai versions
): # extends with image generation fields (text_tokens, image_tokens)
text_tokens: Optional[int] = None
"""Text tokens sent to the model."""

1095
litellm/utils.py
View file

File diff suppressed because it is too large Load diff

460
model_prices_and_context_window.json
View file

@ -249,6 +249,30 @@
"/v1/images/generations"
]
},
"aiml/google/imagen-4.0-ultra-generate-001": {
"litellm_provider": "aiml",
"metadata": {
"notes": "Imagen 4.0 Ultra Generate API - Photorealistic image generation with precise text rendering"
},
"mode": "image_generation",
"output_cost_per_image": 0.063,
"source": "https://docs.aimlapi.com/api-references/image-models/google/imagen-4-ultra-generate",
"supported_endpoints": [
"/v1/images/generations"
]
},
"aiml/google/nano-banana-pro": {
"litellm_provider": "aiml",
"metadata": {
"notes": "Gemini 3 Pro Image (Nano Banana Pro) - Advanced text-to-image generation with reasoning and 4K resolution support"
},
"mode": "image_generation",
"output_cost_per_image": 0.1575,
"source": "https://docs.aimlapi.com/api-references/image-models/google/gemini-3-pro-image-preview",
"supported_endpoints": [
"/v1/images/generations"
]
},
"amazon.nova-canvas-v1:0": {
"litellm_provider": "bedrock",
"max_input_tokens": 2600,
@ -3639,12 +3663,16 @@
"supports_web_search": true
},
"azure/gpt-image-1": {
"input_cost_per_pixel": 4.0054321e-08,
"cache_read_input_image_token_cost": 2.5e-06,
"cache_read_input_token_cost": 1.25e-06,
"input_cost_per_image_token": 1e-05,
"input_cost_per_token": 5e-06,
"litellm_provider": "azure",
"mode": "image_generation",
"output_cost_per_pixel": 0.0,
"output_cost_per_image_token": 4e-05,
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"azure/hd/1024-x-1024/dall-e-3": {
@ -3747,12 +3775,16 @@
]
},
"azure/gpt-image-1-mini": {
"input_cost_per_pixel": 8.0566406e-09,
"cache_read_input_image_token_cost": 2.5e-07,
"cache_read_input_token_cost": 2e-07,
"input_cost_per_image_token": 2.5e-06,
"input_cost_per_token": 2e-06,
"litellm_provider": "azure",
"mode": "image_generation",
"output_cost_per_pixel": 0.0,
"output_cost_per_image_token": 8e-06,
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"azure/gpt-image-1.5": {
@ -10919,13 +10951,13 @@
"supports_tool_choice": true
},
"fireworks_ai/accounts/fireworks/models/deepseek-v3p2": {
"input_cost_per_token": 1.2e-06,
"input_cost_per_token": 5.6e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.2e-06,
"output_cost_per_token": 1.68e-06,
"source": "https://fireworks.ai/models/fireworks/deepseek-v3p2",
"supports_function_calling": true,
"supports_reasoning": true,
@ -11608,6 +11640,7 @@
"supports_tool_choice": true
},
"gemini-1.5-flash": {
"deprecation_date": "2025-09-29",
"input_cost_per_audio_per_second": 2e-06,
"input_cost_per_audio_per_second_above_128k_tokens": 4e-06,
"input_cost_per_character": 1.875e-08,
@ -11712,6 +11745,7 @@
"supports_vision": true
},
"gemini-1.5-flash-exp-0827": {
"deprecation_date": "2025-09-29",
"input_cost_per_audio_per_second": 2e-06,
"input_cost_per_audio_per_second_above_128k_tokens": 4e-06,
"input_cost_per_character": 1.875e-08,
@ -11746,6 +11780,7 @@
"supports_vision": true
},
"gemini-1.5-flash-preview-0514": {
"deprecation_date": "2025-09-29",
"input_cost_per_audio_per_second": 2e-06,
"input_cost_per_audio_per_second_above_128k_tokens": 4e-06,
"input_cost_per_character": 1.875e-08,
@ -11779,6 +11814,7 @@
"supports_vision": true
},
"gemini-1.5-pro": {
"deprecation_date": "2025-09-29",
"input_cost_per_audio_per_second": 3.125e-05,
"input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05,
"input_cost_per_character": 3.125e-07,
@ -11866,6 +11902,7 @@
"supports_vision": true
},
"gemini-1.5-pro-preview-0215": {
"deprecation_date": "2025-09-29",
"input_cost_per_audio_per_second": 3.125e-05,
"input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05,
"input_cost_per_character": 3.125e-07,
@ -11893,6 +11930,7 @@
"supports_tool_choice": true
},
"gemini-1.5-pro-preview-0409": {
"deprecation_date": "2025-09-29",
"input_cost_per_audio_per_second": 3.125e-05,
"input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05,
"input_cost_per_character": 3.125e-07,
@ -11919,6 +11957,7 @@
"supports_tool_choice": true
},
"gemini-1.5-pro-preview-0514": {
"deprecation_date": "2025-09-29",
"input_cost_per_audio_per_second": 3.125e-05,
"input_cost_per_audio_per_second_above_128k_tokens": 6.25e-05,
"input_cost_per_character": 3.125e-07,
@ -12190,6 +12229,7 @@
"tpm": 250000
},
"gemini-2.0-flash-preview-image-generation": {
"deprecation_date": "2025-11-14",
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_audio_token": 7e-07,
"input_cost_per_token": 1e-07,
@ -12228,6 +12268,7 @@
"supports_web_search": true
},
"gemini-2.0-flash-thinking-exp": {
"deprecation_date": "2025-12-02",
"cache_read_input_token_cost": 0.0,
"input_cost_per_audio_per_second": 0,
"input_cost_per_audio_per_second_above_128k_tokens": 0,
@ -12276,6 +12317,7 @@
"supports_web_search": true
},
"gemini-2.0-flash-thinking-exp-01-21": {
"deprecation_date": "2025-12-02",
"cache_read_input_token_cost": 0.0,
"input_cost_per_audio_per_second": 0,
"input_cost_per_audio_per_second_above_128k_tokens": 0,
@ -12462,6 +12504,7 @@
"tpm": 8000000
},
"gemini-2.5-flash-image-preview": {
"deprecation_date": "2026-01-15",
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 3e-07,
@ -12772,6 +12815,7 @@
"tpm": 8000000
},
"gemini-2.5-flash-lite-preview-06-17": {
"deprecation_date": "2025-11-18",
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_audio_token": 5e-07,
"input_cost_per_token": 1e-07,
@ -12861,6 +12905,7 @@
"supports_web_search": true
},
"gemini-2.5-flash-preview-05-20": {
"deprecation_date": "2025-11-18",
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 3e-07,
@ -13132,6 +13177,7 @@
"supports_web_search": true
},
"gemini-2.5-pro-preview-03-25": {
"deprecation_date": "2025-12-02",
"cache_read_input_token_cost": 3.125e-07,
"input_cost_per_audio_token": 1.25e-06,
"input_cost_per_token": 1.25e-06,
@ -13177,6 +13223,7 @@
"supports_web_search": true
},
"gemini-2.5-pro-preview-05-06": {
"deprecation_date": "2025-12-02",
"cache_read_input_token_cost": 3.125e-07,
"input_cost_per_audio_token": 1.25e-06,
"input_cost_per_token": 1.25e-06,
@ -13392,6 +13439,7 @@
"tpm": 10000000
},
"gemini/gemini-1.5-flash": {
"deprecation_date": "2025-09-29",
"input_cost_per_token": 7.5e-08,
"input_cost_per_token_above_128k_tokens": 1.5e-07,
"litellm_provider": "gemini",
@ -13475,6 +13523,7 @@
"tpm": 4000000
},
"gemini/gemini-1.5-flash-8b": {
"deprecation_date": "2025-09-29",
"input_cost_per_token": 0,
"input_cost_per_token_above_128k_tokens": 0,
"litellm_provider": "gemini",
@ -13501,6 +13550,7 @@
"tpm": 4000000
},
"gemini/gemini-1.5-flash-8b-exp-0827": {
"deprecation_date": "2025-09-29",
"input_cost_per_token": 0,
"input_cost_per_token_above_128k_tokens": 0,
"litellm_provider": "gemini",
@ -13526,6 +13576,7 @@
"tpm": 4000000
},
"gemini/gemini-1.5-flash-8b-exp-0924": {
"deprecation_date": "2025-09-29",
"input_cost_per_token": 0,
"input_cost_per_token_above_128k_tokens": 0,
"litellm_provider": "gemini",
@ -13552,6 +13603,7 @@
"tpm": 4000000
},
"gemini/gemini-1.5-flash-exp-0827": {
"deprecation_date": "2025-09-29",
"input_cost_per_token": 0,
"input_cost_per_token_above_128k_tokens": 0,
"litellm_provider": "gemini",
@ -13577,6 +13629,7 @@
"tpm": 4000000
},
"gemini/gemini-1.5-flash-latest": {
"deprecation_date": "2025-09-29",
"input_cost_per_token": 7.5e-08,
"input_cost_per_token_above_128k_tokens": 1.5e-07,
"litellm_provider": "gemini",
@ -13603,6 +13656,7 @@
"tpm": 4000000
},
"gemini/gemini-1.5-pro": {
"deprecation_date": "2025-09-29",
"input_cost_per_token": 3.5e-06,
"input_cost_per_token_above_128k_tokens": 7e-06,
"litellm_provider": "gemini",
@ -13664,6 +13718,7 @@
"tpm": 4000000
},
"gemini/gemini-1.5-pro-exp-0801": {
"deprecation_date": "2025-09-29",
"input_cost_per_token": 3.5e-06,
"input_cost_per_token_above_128k_tokens": 7e-06,
"litellm_provider": "gemini",
@ -13683,6 +13738,7 @@
"tpm": 4000000
},
"gemini/gemini-1.5-pro-exp-0827": {
"deprecation_date": "2025-09-29",
"input_cost_per_token": 0,
"input_cost_per_token_above_128k_tokens": 0,
"litellm_provider": "gemini",
@ -13702,6 +13758,7 @@
"tpm": 4000000
},
"gemini/gemini-1.5-pro-latest": {
"deprecation_date": "2025-09-29",
"input_cost_per_token": 3.5e-06,
"input_cost_per_token_above_128k_tokens": 7e-06,
"litellm_provider": "gemini",
@ -13884,6 +13941,7 @@
"tpm": 4000000
},
"gemini/gemini-2.0-flash-lite-preview-02-05": {
"deprecation_date": "2025-12-02",
"cache_read_input_token_cost": 1.875e-08,
"input_cost_per_audio_token": 7.5e-08,
"input_cost_per_token": 7.5e-08,
@ -13921,6 +13979,7 @@
"tpm": 10000000
},
"gemini/gemini-2.0-flash-live-001": {
"deprecation_date": "2025-12-09",
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_audio_token": 2.1e-06,
"input_cost_per_image": 2.1e-06,
@ -13969,6 +14028,7 @@
"tpm": 250000
},
"gemini/gemini-2.0-flash-preview-image-generation": {
"deprecation_date": "2025-11-14",
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_audio_token": 7e-07,
"input_cost_per_token": 1e-07,
@ -14008,6 +14068,7 @@
"tpm": 10000000
},
"gemini/gemini-2.0-flash-thinking-exp": {
"deprecation_date": "2025-12-02",
"cache_read_input_token_cost": 0.0,
"input_cost_per_audio_per_second": 0,
"input_cost_per_audio_per_second_above_128k_tokens": 0,
@ -14057,6 +14118,7 @@
"tpm": 4000000
},
"gemini/gemini-2.0-flash-thinking-exp-01-21": {
"deprecation_date": "2025-12-02",
"cache_read_input_token_cost": 0.0,
"input_cost_per_audio_per_second": 0,
"input_cost_per_audio_per_second_above_128k_tokens": 0,
@ -14245,6 +14307,7 @@
"tpm": 8000000
},
"gemini/gemini-2.5-flash-image-preview": {
"deprecation_date": "2026-01-15",
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 3e-07,
@ -14565,6 +14628,7 @@
"tpm": 250000
},
"gemini/gemini-2.5-flash-lite-preview-06-17": {
"deprecation_date": "2025-11-18",
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_audio_token": 5e-07,
"input_cost_per_token": 1e-07,
@ -14656,6 +14720,7 @@
"tpm": 250000
},
"gemini/gemini-2.5-flash-preview-05-20": {
"deprecation_date": "2025-11-18",
"cache_read_input_token_cost": 7.5e-08,
"input_cost_per_audio_token": 1e-06,
"input_cost_per_token": 3e-07,
@ -15002,6 +15067,7 @@
"tpm": 250000
},
"gemini/gemini-2.5-pro-preview-03-25": {
"deprecation_date": "2025-12-02",
"cache_read_input_token_cost": 3.125e-07,
"input_cost_per_audio_token": 7e-07,
"input_cost_per_token": 1.25e-06,
@ -15042,6 +15108,7 @@
"tpm": 10000000
},
"gemini/gemini-2.5-pro-preview-05-06": {
"deprecation_date": "2025-12-02",
"cache_read_input_token_cost": 3.125e-07,
"input_cost_per_audio_token": 7e-07,
"input_cost_per_token": 1.25e-06,
@ -15317,6 +15384,7 @@
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
},
"gemini/imagen-3.0-generate-002": {
"deprecation_date": "2025-11-10",
"litellm_provider": "gemini",
"mode": "image_generation",
"output_cost_per_image": 0.04,
@ -15383,6 +15451,7 @@
]
},
"gemini/veo-3.0-fast-generate-preview": {
"deprecation_date": "2025-11-12",
"litellm_provider": "gemini",
"max_input_tokens": 1024,
"max_tokens": 1024,
@ -15397,6 +15466,7 @@
]
},
"gemini/veo-3.0-generate-preview": {
"deprecation_date": "2025-11-12",
"litellm_provider": "gemini",
"max_input_tokens": 1024,
"max_tokens": 1024,
@ -16956,6 +17026,336 @@
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.034,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.133,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1536/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1536-x-1024/gpt-image-1.5": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"low/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.034,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"medium/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.05,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.133,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"high/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.20,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"standard/1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.009,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1024-x-1536/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"1536-x-1024/gpt-image-1.5-2025-12-16": {
"input_cost_per_image": 0.013,
"litellm_provider": "openai",
"mode": "image_generation",
"supported_endpoints": [
"/v1/images/generations",
"/v1/images/edits"
],
"supports_vision": true,
"supports_pdf_input": true
},
"gpt-5": {
"cache_read_input_token_cost": 1.25e-07,
"cache_read_input_token_cost_flex": 6.25e-08,
@ -17717,16 +18117,16 @@
"supports_vision": true
},
"gpt-image-1": {
"input_cost_per_image": 0.042,
"input_cost_per_pixel": 4.0054321e-08,
"input_cost_per_token": 0.000005,
"input_cost_per_image_token": 0.00001,
"cache_read_input_image_token_cost": 2.5e-06,
"cache_read_input_token_cost": 1.25e-06,
"input_cost_per_image_token": 1e-05,
"input_cost_per_token": 5e-06,
"litellm_provider": "openai",
"mode": "image_generation",
"output_cost_per_pixel": 0.0,
"output_cost_per_token": 0.00004,
"output_cost_per_image_token": 4e-05,
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"gpt-image-1-mini": {
@ -18151,6 +18551,18 @@
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/gemma-7b-it": {
"input_cost_per_token": 5e-08,
"litellm_provider": "groq",
"max_input_tokens": 8192,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 8e-08,
"supports_function_calling": true,
"supports_response_schema": false,
"supports_tool_choice": true
},
"groq/meta-llama/llama-guard-4-12b": {
"input_cost_per_token": 2e-07,
"litellm_provider": "groq",
@ -24752,6 +25164,7 @@
"source": "https://docs.mistral.ai/capabilities/code_generation/"
},
"text-embedding-004": {
"deprecation_date": "2026-01-14",
"input_cost_per_character": 2.5e-08,
"input_cost_per_token": 1e-07,
"litellm_provider": "vertex_ai-embedding-models",
@ -27522,6 +27935,7 @@
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
},
"vertex_ai/imagen-3.0-generate-002": {
"deprecation_date": "2025-11-10",
"litellm_provider": "vertex_ai-image-models",
"mode": "image_generation",
"output_cost_per_image": 0.04,
@ -28032,6 +28446,7 @@
]
},
"vertex_ai/veo-3.0-fast-generate-preview": {
"deprecation_date": "2025-11-12",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,
@ -28046,6 +28461,7 @@
]
},
"vertex_ai/veo-3.0-generate-preview": {
"deprecation_date": "2025-11-12",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,
@ -29275,6 +29691,20 @@
"supports_vision": true,
"supports_web_search": true
},
"zai/glm-4.7": {
"cache_creation_input_token_cost": 0,
"cache_read_input_token_cost": 1.1e-07,
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.2e-06,
"litellm_provider": "zai",
"max_input_tokens": 200000,
"max_output_tokens": 128000,
"mode": "chat",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"source": "https://docs.z.ai/guides/overview/pricing"
},
"zai/glm-4.6": {
"input_cost_per_token": 6e-07,
"output_cost_per_token": 2.2e-06,

72
poetry.lock generated
View file

@ -2273,75 +2273,6 @@ googleapis-common-protos = {version = ">=1.56.0,<2.0.0", extras = ["grpc"]}
grpcio = ">=1.44.0,<2.0.0"
protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0"
[[package]]
name = "grpcio"
version = "1.67.1"
description = "HTTP/2-based RPC framework"
optional = false
python-versions = ">=3.8"
groups = ["main", "dev", "proxy-dev"]
markers = "python_version < \"3.14\""
files = [
{file = "grpcio-1.67.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:8b0341d66a57f8a3119b77ab32207072be60c9bf79760fa609c5609f2deb1f3f"},
{file = "grpcio-1.67.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:f5a27dddefe0e2357d3e617b9079b4bfdc91341a91565111a21ed6ebbc51b22d"},
{file = "grpcio-1.67.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:43112046864317498a33bdc4797ae6a268c36345a910de9b9c17159d8346602f"},
{file = "grpcio-1.67.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9b929f13677b10f63124c1a410994a401cdd85214ad83ab67cc077fc7e480f0"},
{file = "grpcio-1.67.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7d1797a8a3845437d327145959a2c0c47c05947c9eef5ff1a4c80e499dcc6fa"},
{file = "grpcio-1.67.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0489063974d1452436139501bf6b180f63d4977223ee87488fe36858c5725292"},
{file = "grpcio-1.67.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9fd042de4a82e3e7aca44008ee2fb5da01b3e5adb316348c21980f7f58adc311"},
{file = "grpcio-1.67.1-cp310-cp310-win32.whl", hash = "sha256:638354e698fd0c6c76b04540a850bf1db27b4d2515a19fcd5cf645c48d3eb1ed"},
{file = "grpcio-1.67.1-cp310-cp310-win_amd64.whl", hash = "sha256:608d87d1bdabf9e2868b12338cd38a79969eaf920c89d698ead08f48de9c0f9e"},
{file = "grpcio-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:7818c0454027ae3384235a65210bbf5464bd715450e30a3d40385453a85a70cb"},
{file = "grpcio-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ea33986b70f83844cd00814cee4451055cd8cab36f00ac64a31f5bb09b31919e"},
{file = "grpcio-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c7a01337407dd89005527623a4a72c5c8e2894d22bead0895306b23c6695698f"},
{file = "grpcio-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80b866f73224b0634f4312a4674c1be21b2b4afa73cb20953cbbb73a6b36c3cc"},
{file = "grpcio-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fff78ba10d4250bfc07a01bd6254a6d87dc67f9627adece85c0b2ed754fa96"},
{file = "grpcio-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8a23cbcc5bb11ea7dc6163078be36c065db68d915c24f5faa4f872c573bb400f"},
{file = "grpcio-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1a65b503d008f066e994f34f456e0647e5ceb34cfcec5ad180b1b44020ad4970"},
{file = "grpcio-1.67.1-cp311-cp311-win32.whl", hash = "sha256:e29ca27bec8e163dca0c98084040edec3bc49afd10f18b412f483cc68c712744"},
{file = "grpcio-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:786a5b18544622bfb1e25cc08402bd44ea83edfb04b93798d85dca4d1a0b5be5"},
{file = "grpcio-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:267d1745894200e4c604958da5f856da6293f063327cb049a51fe67348e4f953"},
{file = "grpcio-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:85f69fdc1d28ce7cff8de3f9c67db2b0ca9ba4449644488c1e0303c146135ddb"},
{file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f26b0b547eb8d00e195274cdfc63ce64c8fc2d3e2d00b12bf468ece41a0423a0"},
{file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4422581cdc628f77302270ff839a44f4c24fdc57887dc2a45b7e53d8fc2376af"},
{file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7616d2ded471231c701489190379e0c311ee0a6c756f3c03e6a62b95a7146e"},
{file = "grpcio-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8a00efecde9d6fcc3ab00c13f816313c040a28450e5e25739c24f432fc6d3c75"},
{file = "grpcio-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:699e964923b70f3101393710793289e42845791ea07565654ada0969522d0a38"},
{file = "grpcio-1.67.1-cp312-cp312-win32.whl", hash = "sha256:4e7b904484a634a0fff132958dabdb10d63e0927398273917da3ee103e8d1f78"},
{file = "grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc"},
{file = "grpcio-1.67.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:aa0162e56fd10a5547fac8774c4899fc3e18c1aa4a4759d0ce2cd00d3696ea6b"},
{file = "grpcio-1.67.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:beee96c8c0b1a75d556fe57b92b58b4347c77a65781ee2ac749d550f2a365dc1"},
{file = "grpcio-1.67.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:a93deda571a1bf94ec1f6fcda2872dad3ae538700d94dc283c672a3b508ba3af"},
{file = "grpcio-1.67.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e6f255980afef598a9e64a24efce87b625e3e3c80a45162d111a461a9f92955"},
{file = "grpcio-1.67.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e838cad2176ebd5d4a8bb03955138d6589ce9e2ce5d51c3ada34396dbd2dba8"},
{file = "grpcio-1.67.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:a6703916c43b1d468d0756c8077b12017a9fcb6a1ef13faf49e67d20d7ebda62"},
{file = "grpcio-1.67.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:917e8d8994eed1d86b907ba2a61b9f0aef27a2155bca6cbb322430fc7135b7bb"},
{file = "grpcio-1.67.1-cp313-cp313-win32.whl", hash = "sha256:e279330bef1744040db8fc432becc8a727b84f456ab62b744d3fdb83f327e121"},
{file = "grpcio-1.67.1-cp313-cp313-win_amd64.whl", hash = "sha256:fa0c739ad8b1996bd24823950e3cb5152ae91fca1c09cc791190bf1627ffefba"},
{file = "grpcio-1.67.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:178f5db771c4f9a9facb2ab37a434c46cb9be1a75e820f187ee3d1e7805c4f65"},
{file = "grpcio-1.67.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0f3e49c738396e93b7ba9016e153eb09e0778e776df6090c1b8c91877cc1c426"},
{file = "grpcio-1.67.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:24e8a26dbfc5274d7474c27759b54486b8de23c709d76695237515bc8b5baeab"},
{file = "grpcio-1.67.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b6c16489326d79ead41689c4b84bc40d522c9a7617219f4ad94bc7f448c5085"},
{file = "grpcio-1.67.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e6a4dcf5af7bbc36fd9f81c9f372e8ae580870a9e4b6eafe948cd334b81cf3"},
{file = "grpcio-1.67.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:95b5f2b857856ed78d72da93cd7d09b6db8ef30102e5e7fe0961fe4d9f7d48e8"},
{file = "grpcio-1.67.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b49359977c6ec9f5d0573ea4e0071ad278ef905aa74e420acc73fd28ce39e9ce"},
{file = "grpcio-1.67.1-cp38-cp38-win32.whl", hash = "sha256:f5b76ff64aaac53fede0cc93abf57894ab2a7362986ba22243d06218b93efe46"},
{file = "grpcio-1.67.1-cp38-cp38-win_amd64.whl", hash = "sha256:804c6457c3cd3ec04fe6006c739579b8d35c86ae3298ffca8de57b493524b771"},
{file = "grpcio-1.67.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:a25bdea92b13ff4d7790962190bf6bf5c4639876e01c0f3dda70fc2769616335"},
{file = "grpcio-1.67.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cdc491ae35a13535fd9196acb5afe1af37c8237df2e54427be3eecda3653127e"},
{file = "grpcio-1.67.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:85f862069b86a305497e74d0dc43c02de3d1d184fc2c180993aa8aa86fbd19b8"},
{file = "grpcio-1.67.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec74ef02010186185de82cc594058a3ccd8d86821842bbac9873fd4a2cf8be8d"},
{file = "grpcio-1.67.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01f616a964e540638af5130469451cf580ba8c7329f45ca998ab66e0c7dcdb04"},
{file = "grpcio-1.67.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:299b3d8c4f790c6bcca485f9963b4846dd92cf6f1b65d3697145d005c80f9fe8"},
{file = "grpcio-1.67.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:60336bff760fbb47d7e86165408126f1dded184448e9a4c892189eb7c9d3f90f"},
{file = "grpcio-1.67.1-cp39-cp39-win32.whl", hash = "sha256:5ed601c4c6008429e3d247ddb367fe8c7259c355757448d7c1ef7bd4a6739e8e"},
{file = "grpcio-1.67.1-cp39-cp39-win_amd64.whl", hash = "sha256:5db70d32d6703b89912af16d6d45d78406374a8b8ef0d28140351dd0ec610e98"},
{file = "grpcio-1.67.1.tar.gz", hash = "sha256:3dc2ed4cabea4dc14d5e708c2b426205956077cc5de419b4d4079315017e9732"},
]
[package.extras]
protobuf = ["grpcio-tools (>=1.67.1)"]
[[package]]
name = "grpcio"
version = "1.76.0"
@ -2349,7 +2280,6 @@ description = "HTTP/2-based RPC framework"
optional = false
python-versions = ">=3.9"
groups = ["main", "dev", "proxy-dev"]
markers = "python_version >= \"3.14\""
files = [
{file = "grpcio-1.76.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:65a20de41e85648e00305c1bb09a3598f840422e522277641145a32d42dcefcc"},
{file = "grpcio-1.76.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:40ad3afe81676fd9ec6d9d406eda00933f218038433980aa19d401490e46ecde"},
@ -8051,4 +7981,4 @@ utils = ["numpydoc"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.9,<4.0"
content-hash = "b010d9da7f5a765670932b78d720aae4fcb819daba050683ee125b4367972419"
content-hash = "7eed2b2c25173a275ac83c55fd901b9b84663b1d7daa54f0e78b30bf1c8f0e3e"

2
pyproject.toml
View file

@ -72,7 +72,7 @@ soundfile = {version = "^0.12.1", optional = true}
# - 1.68.0-1.68.1 has reconnect bug (https://github.com/grpc/grpc/issues/38290)
# - 1.75.0+ has Python 3.14 wheels and bug fix
grpcio = [
{version = ">=1.62.3,<1.68.0", python = "<3.14"},
{version = ">=1.62.3,!=1.68.*,!=1.69.*,!=1.70.*,!=1.71.0,!=1.71.1,!=1.72.0,!=1.72.1,!=1.73.0", python = "<3.14"},
{version = ">=1.75.0", python = ">=3.14"},
]

2
requirements.txt
View file

@ -41,7 +41,7 @@ opentelemetry-api==1.25.0
opentelemetry-sdk==1.25.0
opentelemetry-exporter-otlp==1.25.0
# grpcio: 1.68.0-1.68.1 has reconnect bug (#38290), 1.75+ has Python 3.14 wheels + fix
grpcio>=1.62.3,<1.68.0; python_version < "3.14"
grpcio>=1.62.3,!=1.68.*,!=1.69.*,!=1.70.*,!=1.71.0,!=1.71.1,!=1.72.0,!=1.72.1,!=1.73.0; python_version < "3.14"
grpcio>=1.75.0; python_version >= "3.14"
sentry_sdk==2.21.0 # for sentry error handling
detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests

3
schema.prisma
View file

@ -208,6 +208,9 @@ model LiteLLM_MCPServerTable {
command String?
args String[] @default([])
env Json? @default("{}")
authorization_url String?
token_url String?
registration_url String?
}
// Generate Tokens for Proxy

616
tests/code_coverage_tests/memory_test.py Normal file
View file

@ -0,0 +1,616 @@
"""
Memory Violation Detection Test
Detects bad memory patterns in the LiteLLM codebase that can lead to memory leaks or OOMs.
The detector uses a modular pattern-based system. To add detection for new memory patterns:
1. Create a Pattern subclass implementing get_pattern_name(), visit_assign(), and check_cleanup()
- You can extend the Pattern class with additional methods as needed for your detection logic
2. Add the pattern to MemoryViolationDetector.DEFAULT_PATTERNS
Currently detects:
- queue.get() / queue.get_nowait() operations where variables aren't set to None
- Class-level data structures that have add operations during runtime without size limits:
* Built-in: list, dict, set
* Collections: deque, defaultdict, Counter, OrderedDict, ChainMap
* Queues: queue.Queue, asyncio.Queue (if unbounded, i.e., no maxsize parameter)
* Heap operations: heapq.heappush(), heapq.heapreplace(), heapq.heappushpop() on class-level lists
"""
import ast
import os
from abc import ABC, abstractmethod
from typing import List, Dict, Any, Optional, Sequence
class Pattern(ABC):
"""Base class for memory violation detection patterns"""
@abstractmethod
def get_pattern_name(self) -> str:
"""Return unique identifier for this violation type"""
pass
@abstractmethod
def visit_assign(self, node: ast.Assign, context: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Detect memory-sensitive operations in assignment. Returns list of {line, var_name, call} dicts."""
pass
@abstractmethod
def check_cleanup(self, operations: List[Dict[str, Any]], function_body: List[ast.stmt],
context: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Verify variables are set to None. Returns list of violation dicts."""
pass
class QueueGetPattern(Pattern):
"""Detects queue.get()/get_nowait() operations that aren't cleared"""
def get_pattern_name(self) -> str:
return "queue_reference_not_cleared"
def visit_assign(self, node: ast.Assign, context: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Detect queue.get() or queue.get_nowait() calls where object name contains 'queue'"""
operations = []
if isinstance(node.value, ast.Call):
func = node.value.func
if isinstance(func, ast.Attribute) and func.attr in ("get", "get_nowait"):
obj_name = context["get_attr_string"](func.value)
if "queue" in obj_name.lower() and node.targets and isinstance(node.targets[0], ast.Name):
operations.append({
"line": node.lineno,
"var_name": node.targets[0].id,
"call": context["get_call_string"](node.value),
})
return operations
def check_cleanup(self, operations: List[Dict[str, Any]], function_body: List[ast.stmt],
context: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Flag queue variables that aren't set to None"""
violations = []
is_var_set_to_none = context["is_var_set_to_none"]
current_function = context["current_function"]
file_path = context["file_path"]
queue_vars = {op["var_name"]: op["line"] for op in operations}
for var_name, line_num in queue_vars.items():
if not is_var_set_to_none(var_name, function_body):
violations.append({
"line": line_num,
"type": self.get_pattern_name(),
"var_name": var_name,
"function": current_function,
"file_path": file_path,
"message": (
f"Queue variable '{var_name}' in function "
f"'{current_function}' is not set to None after use. "
f"If the runtime is overwhelmed, this can cause OOM (Out of Memory) errors."
),
})
return violations
class UnboundedDataStructurePattern(Pattern):
"""Detects class-level data structures (lists, dicts, sets) that can grow unbounded"""
def get_pattern_name(self) -> str:
return "unbounded_data_structure"
def visit_assign(self, node: ast.Assign, context: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Detect list/dict/set creations that are at class level"""
operations = []
# Check if this is a data structure creation
is_data_structure = False
structure_type = None
if isinstance(node.value, (ast.List, ast.Dict, ast.Set)):
is_data_structure = True
if isinstance(node.value, ast.List):
structure_type = "list"
elif isinstance(node.value, ast.Dict):
structure_type = "dict"
elif isinstance(node.value, ast.Set):
structure_type = "set"
elif isinstance(node.value, ast.Call):
# Check for list(), dict(), set() calls
func = node.value.func
if isinstance(func, ast.Name):
if func.id in ("list", "dict", "set"):
is_data_structure = True
structure_type = func.id
elif isinstance(func, ast.Attribute):
# Handle cases like collections.defaultdict(list), collections.deque(), etc.
obj_name = context["get_attr_string"](func.value)
attr_name = func.attr
# Check for collections module data structures
if "collections" in obj_name.lower() or "collections" in str(func.value):
if attr_name in ("deque", "defaultdict", "Counter", "OrderedDict", "ChainMap"):
# For deque, we track it and let size checks determine if it's bounded
# (deque with maxlen parameter is bounded, but we detect that via size checks)
is_data_structure = True
structure_type = attr_name
elif attr_name in ("list", "dict", "set"):
# collections.defaultdict(list) pattern
is_data_structure = True
structure_type = "defaultdict" if "defaultdict" in obj_name.lower() else attr_name
# Check for queue.Queue, asyncio.Queue (if unbounded)
elif "queue" in obj_name.lower() or "asyncio" in obj_name.lower():
if attr_name == "Queue":
# Check if maxsize is set (bounded queue)
has_maxsize = False
for keyword in node.value.keywords:
if keyword.arg == "maxsize":
has_maxsize = True
break
if not has_maxsize:
is_data_structure = True
structure_type = "queue"
# Direct attribute access like deque(), Counter(), etc.
elif attr_name in ("deque", "defaultdict", "Counter", "OrderedDict", "ChainMap"):
is_data_structure = True
structure_type = attr_name
if is_data_structure and node.targets and isinstance(node.targets[0], ast.Name):
scope = context.get("current_scope", "function")
# Only track if it's at class level (not module level)
if scope == "class":
operations.append({
"line": node.lineno,
"var_name": node.targets[0].id,
"structure_type": structure_type,
"scope": scope,
"call": context["get_call_string"](node.value) if isinstance(node.value, ast.Call) else f"{structure_type}()",
})
return operations
def check_cleanup(self, operations: List[Dict[str, Any]], function_body: List[ast.stmt],
context: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Flag persistent data structures that have add operations without size limits"""
violations = []
current_function = context["current_function"]
current_scope = context.get("current_scope", "function")
file_path = context["file_path"]
get_attr_string = context["get_attr_string"]
# Skip if this is initialization code (module-level, class-level, or __init__ methods)
# Only flag operations in regular methods/functions that can be called during runtime
is_initialization = (
current_scope in ("module", "class") or
current_function in ("__init__", "__new__", "__class_init__") or
current_function is None # Module-level code
)
if is_initialization:
return violations # Don't flag initialization code
# Track which variables have add operations and size checks
var_add_operations = {} # var_name -> list of lines with add operations
var_size_checks = {} # var_name -> has size limit check
# Build a set of variable names to check
tracked_vars = {op["var_name"]: op for op in operations}
# Scan body for operations on these variables
for stmt in function_body:
for node in ast.walk(stmt):
# Check for method calls that add items
if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
attr_name = node.func.attr
obj_name = get_attr_string(node.func.value)
# Check if this is an add operation on one of our tracked variables
for var_name, op in tracked_vars.items():
structure_type = op["structure_type"]
# Match variable name (exact or as attribute)
if obj_name == var_name or obj_name.endswith(f".{var_name}") or obj_name.endswith(f"['{var_name}']"):
# Check for add operations
add_ops = {
"list": ["append", "extend", "insert"],
"dict": ["update", "setdefault"],
"set": ["add", "update"],
"deque": ["append", "appendleft", "extend", "extendleft", "insert"],
"defaultdict": ["update", "setdefault"],
"Counter": ["update"],
"OrderedDict": ["update", "setdefault"],
"ChainMap": ["new_child"],
"queue": ["put", "put_nowait"],
}
if attr_name in add_ops.get(structure_type, []):
if var_name not in var_add_operations:
var_add_operations[var_name] = []
var_add_operations[var_name].append(node.lineno)
# Check for size limit checks (len() calls, maxsize/maxlen attributes)
if (attr_name in ("__len__",) or
"maxsize" in attr_name.lower() or
"max_size" in attr_name.lower() or
attr_name == "maxlen"): # For deque
var_size_checks[var_name] = True
# Check for heapq operations on tracked lists (heapq.heappush, heapq.heappop)
if isinstance(node, ast.Call):
func = node.func
# Check for heapq.heappush(list_var, item) or heapq.heappop(list_var)
if isinstance(func, ast.Attribute):
func_obj = get_attr_string(func.value)
func_name = func.attr
# Check if it's a heapq operation
if func_obj == "heapq" and func_name in ("heappush", "heapreplace", "heappushpop"):
# First argument should be our tracked variable
if len(node.args) > 0:
arg_name = get_attr_string(node.args[0])
for var_name, op in tracked_vars.items():
if op["structure_type"] == "list" and (
arg_name == var_name or arg_name.endswith(f".{var_name}")
):
if var_name not in var_add_operations:
var_add_operations[var_name] = []
var_add_operations[var_name].append(node.lineno)
# Check for dict item assignment: dict[key] = value
if isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Subscript):
target_name = get_attr_string(target.value)
for var_name in tracked_vars:
if target_name == var_name or target_name.endswith(f".{var_name}"):
if var_name not in var_add_operations:
var_add_operations[var_name] = []
var_add_operations[var_name].append(node.lineno)
# Check for augmented assignment: list += [...]
if isinstance(node, ast.AugAssign):
target_name = get_attr_string(node.target)
for var_name in tracked_vars:
if target_name == var_name or target_name.endswith(f".{var_name}"):
if var_name not in var_add_operations:
var_add_operations[var_name] = []
var_add_operations[var_name].append(node.lineno)
# Check for size comparisons in conditionals
if isinstance(node, (ast.If, ast.While, ast.Assert)):
test = getattr(node, "test", None)
if test:
for comp_node in ast.walk(test):
if isinstance(comp_node, ast.Compare):
left_str = get_attr_string(comp_node.left) if hasattr(comp_node, "left") else ""
# Check for len() calls
if isinstance(comp_node.left, ast.Call):
call_func = comp_node.left.func
if isinstance(call_func, ast.Name) and call_func.id == "len":
if len(comp_node.left.args) > 0:
arg_name = get_attr_string(comp_node.left.args[0])
for var_name in tracked_vars:
if arg_name == var_name or arg_name.endswith(f".{var_name}"):
# Check if comparing to a limit
for comparator in comp_node.comparators:
if isinstance(comparator, ast.Constant):
var_size_checks[var_name] = True
elif isinstance(comparator, ast.Name):
# Could be a constant like MAX_SIZE
if "max" in comparator.id.lower() or "limit" in comparator.id.lower():
var_size_checks[var_name] = True
# Handle deprecated ast.Num for Python < 3.8
try:
Num = getattr(ast, "Num", None)
if Num and isinstance(comparator, Num):
var_size_checks[var_name] = True
except (AttributeError, TypeError):
pass
# Check for direct variable comparisons
for var_name in tracked_vars:
if var_name in left_str:
for comparator in comp_node.comparators:
if isinstance(comparator, ast.Constant):
var_size_checks[var_name] = True
# Handle deprecated ast.Num for Python < 3.8
try:
Num = getattr(ast, "Num", None)
if Num and isinstance(comparator, Num):
var_size_checks[var_name] = True
except (AttributeError, TypeError):
pass
# Flag violations: persistent structures with add operations but no size checks
for op in operations:
var_name = op["var_name"]
structure_type = op["structure_type"]
if var_name in var_add_operations and var_name not in var_size_checks:
violations.append({
"line": op["line"],
"type": self.get_pattern_name(),
"var_name": var_name,
"function": current_function or "class-level",
"file_path": file_path,
"message": (
f"Class-level {structure_type} '{var_name}' "
f"has add operations (lines {var_add_operations[var_name]}) but no size limit checks. "
f"This can lead to unbounded memory growth and OOM errors during runtime."
),
})
return violations
class MemoryViolationDetector(ast.NodeVisitor):
"""AST visitor that detects memory violations using registered patterns"""
DEFAULT_PATTERNS: List[Pattern] = [QueueGetPattern(), UnboundedDataStructurePattern()]
def __init__(self, file_path: str, patterns: Optional[Sequence[Pattern]] = None):
self.file_path = file_path
self.violations: List[Dict[str, Any]] = []
self.current_function: Optional[str] = None
self.current_scope: str = "module" # Track current scope: module, class, function
self.patterns = self.DEFAULT_PATTERNS if patterns is None else patterns
self.ast_tree: Optional[ast.Module] = None # Store full AST for module-level checks
self.pattern_operations: Dict[str, List[Dict[str, Any]]] = {
pattern.get_pattern_name(): [] for pattern in self.patterns
}
# Track class-level operations separately (for checking in functions)
self.class_level_operations: Dict[str, List[Dict[str, Any]]] = {
pattern.get_pattern_name(): [] for pattern in self.patterns
}
self._context = {
"get_call_string": self._get_call_string,
"get_attr_string": self._get_attr_string,
"is_var_set_to_none": self._is_var_set_to_none,
"current_function": None,
"current_scope": "module",
"file_path": file_path,
}
def visit_ClassDef(self, node):
"""Track class scope"""
old_scope = self.current_scope
self.current_scope = "class"
self._context["current_scope"] = "class"
self.generic_visit(node)
self.current_scope = old_scope
self._context["current_scope"] = old_scope
def visit_FunctionDef(self, node):
"""Track function scope and check cleanup after visiting"""
old_function = self.current_function
old_scope = self.current_scope
self.current_function = node.name
self.current_scope = "function"
self._context["current_function"] = node.name
self._context["current_scope"] = "function"
for pattern_name in self.pattern_operations:
self.pattern_operations[pattern_name] = []
self.generic_visit(node)
self._check_function_cleanup(node)
self.current_function = old_function
self.current_scope = old_scope
self._context["current_function"] = old_function
self._context["current_scope"] = old_scope
def visit_AsyncFunctionDef(self, node):
"""Track async function scope and check cleanup after visiting"""
old_function = self.current_function
old_scope = self.current_scope
self.current_function = node.name
self.current_scope = "function"
self._context["current_function"] = node.name
self._context["current_scope"] = "function"
for pattern_name in self.pattern_operations:
self.pattern_operations[pattern_name] = []
self.generic_visit(node)
self._check_function_cleanup(node)
self.current_function = old_function
self.current_scope = old_scope
self._context["current_function"] = old_function
self._context["current_scope"] = old_scope
def visit_Assign(self, node):
"""Detect memory-sensitive operations in assignments"""
for pattern in self.patterns:
operations = pattern.visit_assign(node, self._context)
# Track function-level operations
self.pattern_operations[pattern.get_pattern_name()].extend(operations)
# Track class-level operations separately (for checking in functions)
for op in operations:
if op.get("scope") == "class":
self.class_level_operations[pattern.get_pattern_name()].append(op)
self.generic_visit(node)
def _check_function_cleanup(self, node):
"""Check cleanup for all detected operations"""
for pattern in self.patterns:
operations = self.pattern_operations[pattern.get_pattern_name()]
if operations:
violations = pattern.check_cleanup(operations, node.body, self._context)
self.violations.extend(violations)
# For UnboundedDataStructurePattern, also check if this function modifies class-level structures
if isinstance(pattern, UnboundedDataStructurePattern):
class_ops = self.class_level_operations[pattern.get_pattern_name()]
if class_ops and self.current_function not in ("__init__", "__new__", "__class_init__", None):
# Check if this regular function modifies class-level structures
violations = pattern.check_cleanup(class_ops, node.body, self._context)
self.violations.extend(violations)
def _check_module_level_cleanup(self):
"""Check cleanup for module/class level operations"""
# Module-level operations are now checked when visiting functions
# This method is kept for potential future use but doesn't need to do anything
# since we only want to flag runtime modifications in functions, not initialization code
pass
def _is_var_set_to_none(self, var_name: str, body: List[ast.stmt]) -> bool:
"""Check if variable is set to None after its initial assignment"""
assignment_line = None
for stmt in body:
for node in ast.walk(stmt):
if isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Name) and target.id == var_name:
assignment_line = node.lineno
break
if assignment_line:
break
if assignment_line:
break
if not assignment_line:
return False
for stmt in body:
for node in ast.walk(stmt):
if isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Name) and target.id == var_name and node.lineno > assignment_line:
if isinstance(node.value, ast.Constant) and node.value.value is None:
return True
try:
NameConstant = getattr(ast, "NameConstant", None)
if NameConstant and isinstance(node.value, NameConstant):
if getattr(node.value, "value", None) is None:
return True
except (AttributeError, TypeError):
pass
return False
def _get_call_string(self, node: ast.Call) -> str:
"""Get string representation of function call"""
try:
if hasattr(ast, "unparse"):
return ast.unparse(node)
elif isinstance(node.func, ast.Attribute):
return f"{self._get_attr_string(node.func.value)}.{node.func.attr}()"
return str(node)
except Exception:
return str(node)
def _get_attr_string(self, node: ast.AST) -> str:
"""Get string representation of attribute access"""
if isinstance(node, ast.Name):
return node.id
elif isinstance(node, ast.Attribute):
return f"{self._get_attr_string(node.value)}.{node.attr}"
return str(node)
def check_file_for_memory_violations(file_path: str, patterns: Optional[Sequence[Pattern]] = None) -> List[Dict[str, Any]]:
"""Check a single file for memory violations"""
try:
with open(file_path, "r", encoding="utf-8") as f:
content = f.read()
if "test" in file_path.lower() or "__pycache__" in file_path:
return []
tree = ast.parse(content, filename=file_path)
detector = MemoryViolationDetector(file_path, patterns)
detector.ast_tree = tree # Store AST for potential future use
detector.visit(tree)
# Class-level operations are checked when visiting functions
return detector.violations
except Exception as e:
print(f"Error parsing {file_path}: {e}")
return []
def check_directory_for_memory_violations(directory_path: str, ignore_patterns: Optional[List[str]] = None,
patterns: Optional[Sequence[Pattern]] = None) -> List[Dict[str, Any]]:
"""Recursively scan directory for memory violations"""
if ignore_patterns is None:
ignore_patterns = ["__pycache__", ".pyc", "site-packages", "venv", ".venv", "env", ".env", "node_modules", "tests"]
all_violations = []
for root, _dirs, files in os.walk(directory_path):
if any(pattern in root for pattern in ignore_patterns):
continue
for file in files:
if file.endswith(".py"):
violations = check_file_for_memory_violations(os.path.join(root, file), patterns)
all_violations.extend(violations)
return all_violations
def main():
"""Run memory violation detection on codebase"""
codebase_path = "./litellm"
print("=" * 80)
print("MEMORY VIOLATION DETECTION TEST")
print("=" * 80)
print(f"Scanning: {codebase_path}")
print(f"Active patterns: {', '.join(p.get_pattern_name() for p in MemoryViolationDetector.DEFAULT_PATTERNS)}")
print()
violations = check_directory_for_memory_violations(codebase_path)
if violations:
by_type = {}
for v in violations:
vtype = v["type"]
if vtype not in by_type:
by_type[vtype] = []
by_type[vtype].append(v)
print("MEMORY VIOLATIONS FOUND:")
print("=" * 80)
total = len(violations)
for vtype, vlist in by_type.items():
print(f"\n{vtype.upper().replace('_', ' ')}: {len(vlist)} violation(s)")
print("-" * 80)
for v in vlist[:10]:
print(f" [VIOLATION] {v['file_path'] if 'file_path' in v else 'unknown'}:{v['line']}")
print(f" Function: {v['function']}")
print(f" Variable: {v['var_name']}")
print(f" {v['message']}")
print()
if len(vlist) > 10:
print(f" ... and {len(vlist) - 10} more violations of this type")
print("=" * 80)
print(f"TOTAL VIOLATIONS: {total}")
print()
print("RECOMMENDATIONS:")
print(" 1. Set queue variables to None after use: obj = queue.get(); ...; obj = None")
print(" 2. Use bounded queues to prevent unbounded accumulation")
print(" 3. Process items faster than they're added, or drain queues periodically")
print(" 4. For class-level data structures (lists, dicts, sets) that are modified at runtime:")
print(" - Add size limit checks: if len(data) >= MAX_SIZE: ...")
print(" - Implement periodic cleanup or use bounded collections")
print(" - Consider using collections.deque with maxlen for lists")
print("=" * 80)
first_v = violations[0]
raise Exception(
f"Found {total} memory violations! "
f"First violation: {first_v.get('file_path', 'unknown')}:{first_v['line']} - "
f"{first_v['message']}"
)
else:
print("OK No memory violations found!")
if __name__ == "__main__":
main()

1
tests/code_coverage_tests/recursive_detector.py
View file

@ -37,6 +37,7 @@ IGNORE_FUNCTIONS = [
"_split_text", # max depth set.
"_delete_nested_value_custom", # max depth set (bounded by number of path segments).
"filter_exceptions_from_params", # max depth set (default 20) to prevent infinite recursion.
"__getattr__", # lazy loading pattern in litellm/__init__.py with proper caching to prevent infinite recursion.
]

156
tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py
View file

@ -903,3 +903,159 @@ def test_convert_to_model_response_object_with_thinking_content():
resp: ModelResponse = convert_to_model_response_object(**args)
assert resp is not None
assert resp.choices[0].message.reasoning_content is not None
def test_convert_to_model_response_object_with_empty_error_object():
"""
Test that convert_to_model_response_object handles empty error objects gracefully.
This is a regression test for issue #18407 where providers like Apertis return
empty error objects even on successful responses, causing spurious APIErrors.
The error object structure:
{
"error": {
"message": "",
"type": "",
"param": "",
"code": null
}
}
"""
response_object = {
"model": "minimax-m2.1",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Hey! I'm doing well, thanks for asking!",
},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 49,
"completion_tokens": 87,
"total_tokens": 136,
},
"error": {
"message": "",
"type": "",
"param": "",
"code": None,
},
}
# This should NOT raise an exception
result = convert_to_model_response_object(
model_response_object=ModelResponse(),
response_object=response_object,
stream=False,
start_time=datetime.now(),
end_time=datetime.now(),
hidden_params=None,
_response_headers=None,
convert_tool_call_to_json_mode=False,
)
assert isinstance(result, ModelResponse)
assert result.model == "minimax-m2.1"
assert len(result.choices) == 1
assert result.choices[0].message.content == "Hey! I'm doing well, thanks for asking!"
def test_convert_to_model_response_object_with_real_error():
"""
Test that convert_to_model_response_object still raises for real errors.
Ensures the empty error fix doesn't break legitimate error handling.
"""
response_object = {
"error": {
"message": "Rate limit exceeded",
"type": "rate_limit_error",
"param": None,
"code": 429,
},
}
with pytest.raises(Exception) as exc_info:
convert_to_model_response_object(
model_response_object=ModelResponse(),
response_object=response_object,
stream=False,
start_time=datetime.now(),
end_time=datetime.now(),
hidden_params=None,
_response_headers=None,
convert_tool_call_to_json_mode=False,
)
# The exception should have the error message
assert hasattr(exc_info.value, "message")
assert "Rate limit exceeded" in str(exc_info.value.message)
def test_convert_to_model_response_object_with_empty_dict_error():
"""
Test that convert_to_model_response_object handles completely empty error dict.
"""
response_object = {
"model": "test-model",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Hello!",
},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
},
"error": {}, # Completely empty error object
}
# This should NOT raise an exception
result = convert_to_model_response_object(
model_response_object=ModelResponse(),
response_object=response_object,
stream=False,
start_time=datetime.now(),
end_time=datetime.now(),
hidden_params=None,
_response_headers=None,
convert_tool_call_to_json_mode=False,
)
assert isinstance(result, ModelResponse)
assert result.choices[0].message.content == "Hello!"
def test_convert_to_model_response_object_with_error_code_only():
"""
Test that errors with only a code (no message) are still treated as real errors.
"""
response_object = {
"error": {
"message": "",
"code": 500,
},
}
with pytest.raises(Exception):
convert_to_model_response_object(
model_response_object=ModelResponse(),
response_object=response_object,
stream=False,
start_time=datetime.now(),
end_time=datetime.now(),
hidden_params=None,
_response_headers=None,
convert_tool_call_to_json_mode=False,
)

8
tests/local_testing/test_anthropic_prompt_caching.py
View file

@ -104,7 +104,6 @@ async def test_litellm_anthropic_prompt_caching_tools():
],
extra_headers={
"anthropic-version": "2023-06-01",
"anthropic-beta": "prompt-caching-2024-07-31",
},
)
@ -112,11 +111,12 @@ async def test_litellm_anthropic_prompt_caching_tools():
print("call args=", mock_post.call_args)
expected_url = "https://api.anthropic.com/v1/messages"
# Note: anthropic-beta header for prompt-caching is no longer required
# Anthropic now supports prompt caching automatically when cache_control is used
expected_headers = {
"accept": "application/json",
"content-type": "application/json",
"anthropic-version": "2023-06-01",
"anthropic-beta": "prompt-caching-2024-07-31",
"x-api-key": "mock_api_key",
}
@ -285,7 +285,6 @@ async def test_anthropic_api_prompt_caching_basic():
max_tokens=10,
extra_headers={
"anthropic-version": "2023-06-01",
"anthropic-beta": "prompt-caching-2024-07-31",
},
)
@ -356,7 +355,6 @@ async def test_anthropic_api_prompt_caching_basic_with_cache_creation():
max_tokens=10,
extra_headers={
"anthropic-version": "2023-06-01",
"anthropic-beta": "prompt-caching-2024-07-31",
},
)
@ -645,7 +643,6 @@ async def test_litellm_anthropic_prompt_caching_system():
],
extra_headers={
"anthropic-version": "2023-06-01",
"anthropic-beta": "prompt-caching-2024-07-31",
},
)
@ -657,7 +654,6 @@ async def test_litellm_anthropic_prompt_caching_system():
"accept": "application/json",
"content-type": "application/json",
"anthropic-version": "2023-06-01",
"anthropic-beta": "prompt-caching-2024-07-31",
"x-api-key": "mock_api_key",
}

106
tests/local_testing/test_gemini_reasoning_content.py
View file

@ -1,4 +1,5 @@
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
from litellm.llms.vertex_ai.gemini.transformation import _gemini_convert_messages_with_history
def test_thought_true_creates_thinking_block():
@ -36,3 +37,108 @@ def test_thought_signature_without_thought_does_not_create_block():
config = VertexGeminiConfig()
thinking_blocks = config._extract_thinking_blocks_from_parts(parts)
assert thinking_blocks == []
def test_extract_thought_signatures_from_regular_parts():
"""
Test that thoughtSignatures are extracted from regular text parts (without thought=True).
This is the key feature for Gemini 3 multi-turn context preservation.
"""
parts = [{"text": "I am Gemini", "thoughtSignature": "sig-regular-123"}]
config = VertexGeminiConfig()
# Should NOT create thinking block
thinking_blocks = config._extract_thinking_blocks_from_parts(parts)
assert thinking_blocks == []
# Should extract thought signature
signatures = config._extract_thought_signatures_from_parts(parts)
assert signatures is not None
assert len(signatures) == 1
assert signatures[0] == "sig-regular-123"
def test_extract_multiple_thought_signatures():
"""
Test extraction of multiple thoughtSignatures from different parts.
"""
parts = [
{"text": "Part 1", "thoughtSignature": "sig-1"},
{"text": "Part 2", "thoughtSignature": "sig-2"},
{"text": "Part 3"} # No signature
]
config = VertexGeminiConfig()
signatures = config._extract_thought_signatures_from_parts(parts)
assert signatures is not None
assert len(signatures) == 2
assert signatures[0] == "sig-1"
assert signatures[1] == "sig-2"
def test_round_trip_thought_signature_in_conversation():
"""
Test that thoughtSignatures are properly round-tripped through conversation history.
This ensures multi-turn context preservation works correctly.
"""
messages = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"content": "Hi there",
"provider_specific_fields": {
"thought_signatures": ["sig-round-trip-abc"]
}
},
{"role": "user", "content": "How are you?"}
]
gemini_contents = _gemini_convert_messages_with_history(messages)
# Find the assistant (model) message
model_message = None
for content in gemini_contents:
if content.get("role") == "model":
model_message = content
break
assert model_message is not None
assert len(model_message["parts"]) >= 1
# Check that the text part has the thoughtSignature
text_part = model_message["parts"][0]
assert text_part["text"] == "Hi there"
assert "thoughtSignature" in text_part
assert text_part["thoughtSignature"] == "sig-round-trip-abc"
def test_round_trip_without_thought_signature_still_works():
"""
Test that messages without thoughtSignatures continue to work normally.
This ensures backward compatibility.
"""
messages = [
{"role": "user", "content": "Hello"},
{
"role": "assistant",
"content": "Hi there"
},
{"role": "user", "content": "How are you?"}
]
gemini_contents = _gemini_convert_messages_with_history(messages)
# Find the assistant (model) message
model_message = None
for content in gemini_contents:
if content.get("role") == "model":
model_message = content
break
assert model_message is not None
assert len(model_message["parts"]) >= 1
# Check that the text part works without thoughtSignature
text_part = model_message["parts"][0]
assert text_part["text"] == "Hi there"
assert "thoughtSignature" not in text_part

2
tests/logging_callback_tests/langfuse_expected_request_body/completion_with_bedrock_call.json
View file

@ -54,7 +54,7 @@
"id": "time-14-13-16-469836_chatcmpl-3803a9e9-aa68-4493-94d9-247f354830d6",
"endTime": "2025-05-26T14:13:16.795438-07:00",
"completionStartTime": "2025-05-26T14:13:16.795438-07:00",
"model": "anthropic.claude-3-5-sonnet-20240620-v1:0",
"model": "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0",
"modelParameters": {
"aws_region": "us-east-1"
},

250
tests/logging_callback_tests/test_generic_api_callback.py
View file

@ -207,3 +207,253 @@ async def test_generic_api_callback_multiple_logs():
assert (
payload_item["response"]["choices"][0]["message"]["content"] == "hi"
), "Response should be hi"
@pytest.mark.asyncio
async def test_generic_api_callback_ndjson_format():
"""
Test the GenericAPILogger callback with ndjson log format.
Validates that logs are sent as newline-delimited JSON.
"""
# Create a mock for the async_httpx_client's post method
mock_post = AsyncMock()
mock_post.return_value.status_code = 200
mock_post.return_value.text = "OK"
# Set up an endpoint for testing
test_endpoint = "https://example.com/api/logs"
test_headers = {"Authorization": "Bearer test_token"}
os.environ["GENERIC_LOGGER_ENDPOINT"] = test_endpoint
# Initialize the GenericAPILogger with ndjson format
generic_logger = GenericAPILogger(
endpoint=test_endpoint,
headers=test_headers,
flush_interval=1,
log_format="ndjson" # Set NDJSON format
)
generic_logger.async_httpx_client.post = mock_post
litellm.callbacks = [generic_logger]
# Make multiple completion calls to generate multiple logs
for i in range(3):
response = await litellm.acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": f"Hello, world! {i}"}],
mock_response="hi",
user="test_user",
)
# Wait for async flush
await asyncio.sleep(3)
# Assert httpx post was called
mock_post.assert_called_once()
# Get the actual request body from the mock
actual_url = mock_post.call_args[1]["url"]
assert actual_url == test_endpoint, f"Expected URL {test_endpoint}, got {actual_url}"
# Get the data sent
ndjson_data = mock_post.call_args[1]["data"]
print("##########\n")
print("ndjson_data:", ndjson_data)
print("##########\n")
# Validate it's NDJSON format (newline-delimited)
assert isinstance(ndjson_data, str), "Data should be a string for NDJSON"
# Split by newlines and parse each line
lines = ndjson_data.strip().split("\n")
assert len(lines) == 3, f"Expected 3 lines of NDJSON, got {len(lines)}"
# Validate each line is valid JSON
for i, line in enumerate(lines):
payload_item = json.loads(line)
payload_item = StandardLoggingPayload(**payload_item)
# Basic assertions
assert payload_item["response_cost"] > 0, "Response cost should be greater than 0"
assert payload_item["model"] == "gpt-4o", "Model should be gpt-4o"
assert payload_item["model_parameters"]["user"] == "test_user", "User should be test_user"
@pytest.mark.asyncio
async def test_generic_api_callback_single_format():
"""
Test the GenericAPILogger callback with single log format.
Validates that each log is sent as an individual request in parallel.
"""
# Create a mock for the async_httpx_client's post method
mock_post = AsyncMock()
mock_post.return_value.status_code = 200
mock_post.return_value.text = "OK"
# Set up an endpoint for testing
test_endpoint = "https://example.com/api/logs"
test_headers = {"Authorization": "Bearer test_token"}
os.environ["GENERIC_LOGGER_ENDPOINT"] = test_endpoint
# Initialize the GenericAPILogger with single format
generic_logger = GenericAPILogger(
endpoint=test_endpoint,
headers=test_headers,
flush_interval=1, # Quick flush to trigger batch send
log_format="single" # Set single format
)
generic_logger.async_httpx_client.post = mock_post
litellm.callbacks = [generic_logger]
# Make 3 completion calls
for i in range(3):
response = await litellm.acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": f"Hello, world! {i}"}],
mock_response="hi",
user="test_user",
)
# Wait for async flush
await asyncio.sleep(3)
# Assert httpx post was called 3 times (once per log in batch)
assert mock_post.call_count == 3, f"Expected 3 calls, got {mock_post.call_count}"
# Validate each call sent a single log object (not an array)
for call_idx in range(3):
call_args = mock_post.call_args_list[call_idx]
json_data = call_args[1]["data"]
print(f"########## Call {call_idx} ##########")
print("json_data:", json_data)
# Parse and validate - should be a single object, not an array
actual_request = json.loads(json_data)
assert isinstance(actual_request, dict), f"Call {call_idx}: Expected dict, got {type(actual_request)}"
# Validate it's a valid StandardLoggingPayload
payload_item = StandardLoggingPayload(**actual_request)
assert payload_item["response_cost"] > 0, "Response cost should be greater than 0"
assert payload_item["model"] == "gpt-4o", "Model should be gpt-4o"
@pytest.mark.asyncio
async def test_generic_api_callback_json_array_format_explicit():
"""
Test the GenericAPILogger callback with explicit json_array format.
Validates backward compatibility when explicitly set to json_array.
"""
# Create a mock for the async_httpx_client's post method
mock_post = AsyncMock()
mock_post.return_value.status_code = 200
mock_post.return_value.text = "OK"
# Set up an endpoint for testing
test_endpoint = "https://example.com/api/logs"
test_headers = {"Authorization": "Bearer test_token"}
os.environ["GENERIC_LOGGER_ENDPOINT"] = test_endpoint
# Initialize the GenericAPILogger with explicit json_array format
generic_logger = GenericAPILogger(
endpoint=test_endpoint,
headers=test_headers,
flush_interval=1,
log_format="json_array" # Explicitly set json_array
)
generic_logger.async_httpx_client.post = mock_post
litellm.callbacks = [generic_logger]
# Make multiple completion calls
for i in range(5):
response = await litellm.acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": f"Hello, world! {i}"}],
mock_response="hi",
user="test_user",
)
# Wait for async flush
await asyncio.sleep(3)
# Assert httpx post was called once (batched)
mock_post.assert_called_once()
# Get the data and validate it's a JSON array
json_data = mock_post.call_args[1]["data"]
actual_request = json.loads(json_data)
assert isinstance(actual_request, list), "Request body should be a list (JSON array)"
assert len(actual_request) == 5, f"Expected 5 items, got {len(actual_request)}"
# Validate each item
for payload_item in actual_request:
payload_item = StandardLoggingPayload(**payload_item)
assert payload_item["response_cost"] > 0, "Response cost should be greater than 0"
assert payload_item["model"] == "gpt-4o", "Model should be gpt-4o"
@pytest.mark.asyncio
async def test_generic_api_callback_sumologic_uses_ndjson():
"""
Test that the sumologic callback uses ndjson format by default
when loaded from generic_api_compatible_callbacks.json
"""
# Create a mock for the async_httpx_client's post method
mock_post = AsyncMock()
mock_post.return_value.status_code = 200
mock_post.return_value.text = "OK"
# Set environment variable for sumologic
os.environ["SUMOLOGIC_WEBHOOK_URL"] = "https://collectors.sumologic.com/receiver/v1/http/test123"
# Initialize using callback_name (loads from JSON config)
generic_logger = GenericAPILogger(
callback_name="sumologic",
flush_interval=1
)
generic_logger.async_httpx_client.post = mock_post
litellm.callbacks = [generic_logger]
# Verify the logger has ndjson format
assert generic_logger.log_format == "ndjson", "Sumologic should use ndjson format"
# Make completion calls
for i in range(2):
await litellm.acompletion(
model="gpt-4o",
messages=[{"role": "user", "content": f"Test {i}"}],
mock_response="response",
user="test_user",
)
# Wait for async flush
await asyncio.sleep(3)
# Assert httpx post was called
mock_post.assert_called_once()
# Verify NDJSON format
ndjson_data = mock_post.call_args[1]["data"]
assert isinstance(ndjson_data, str), "Data should be a string for NDJSON"
lines = ndjson_data.strip().split("\n")
assert len(lines) == 2, f"Expected 2 lines of NDJSON, got {len(lines)}"
# Each line should be valid JSON
for line in lines:
json.loads(line) # Will raise if invalid JSON
@pytest.mark.asyncio
async def test_generic_api_callback_invalid_log_format():
"""
Test that invalid log_format values raise a ValueError
"""
test_endpoint = "https://example.com/api/logs"
os.environ["GENERIC_LOGGER_ENDPOINT"] = test_endpoint
with pytest.raises(ValueError, match="Invalid log_format"):
GenericAPILogger(
endpoint=test_endpoint,
log_format="invalid_format" # type: ignore # Intentionally invalid for testing
)

13
tests/logging_callback_tests/test_opentelemetry_unit_tests.py
View file

@ -40,11 +40,15 @@ class TestOpentelemetryUnitTests(BaseLoggingCallbackTest):
@pytest.mark.asyncio
async def test_opentelemetry_integration(self):
"""
Unit test to confirm the parent otel span is ended.
Unit test to confirm external parent otel spans are NOT ended by LiteLLM.
External spans (passed via metadata) should be managed by their creators,
not by LiteLLM. This prevents premature closure of spans from Langfuse,
user code, or other external observability tools.
"""
# Reset all callbacks to ensure clean state
litellm.logging_callback_manager._reset_all_callbacks()
parent_otel_span = MagicMock()
litellm.callbacks = ["otel"]
@ -57,8 +61,9 @@ class TestOpentelemetryUnitTests(BaseLoggingCallbackTest):
await asyncio.sleep(1)
# Verify span was ended (may be called multiple times due to callback architecture)
parent_otel_span.end.assert_called()
# Verify external span was NOT ended by LiteLLM
# External spans should only be closed by their creators
parent_otel_span.end.assert_not_called()
def test_init_tracing_respects_existing_tracer_provider(self):
"""

8
tests/mcp_tests/test_mcp_server.py
View file

@ -1057,7 +1057,7 @@ async def test_mcp_server_manager_config_integration_with_database():
)
# Test the add_update_server method (this tests our fix)
await test_manager.add_update_server(db_server)
await test_manager.add_server(db_server)
# Verify the server was added with correct access_groups
registry = test_manager.get_registry()
@ -1381,7 +1381,7 @@ async def test_add_update_server_with_alias():
mock_mcp_server.token_url = None
# Add server to manager
await test_manager.add_update_server(mock_mcp_server)
await test_manager.add_server(mock_mcp_server)
# Verify server was added with correct name (should use alias)
assert "test-server-123" in test_manager.registry
@ -1421,7 +1421,7 @@ async def test_add_update_server_without_alias():
mock_mcp_server.token_url = None
# Add server to manager
await test_manager.add_update_server(mock_mcp_server)
await test_manager.add_server(mock_mcp_server)
# Verify server was added with correct name (should use server_name)
assert "test-server-123" in test_manager.registry
@ -1461,7 +1461,7 @@ async def test_add_update_server_fallback_to_server_id():
mock_mcp_server.token_url = None
# Add server to manager
await test_manager.add_update_server(mock_mcp_server)
await test_manager.add_server(mock_mcp_server)
# Verify server was added with correct name (should use server_id)
assert "test-server-123" in test_manager.registry

1
tests/old_proxy_tests/tests/test_anthropic_context_caching.py
View file

@ -30,7 +30,6 @@ response = client.chat.completions.create(
],
extra_headers={
"anthropic-version": "2023-06-01",
"anthropic-beta": "prompt-caching-2024-07-31",
},
)

8
tests/store_model_in_db_tests/test_mcp_servers.py
View file

@ -139,7 +139,7 @@ async def test_create_mcp_server_direct():
mock_get_prisma.return_value = mock_prisma
# Mock server manager
mock_manager.add_update_server = mock.AsyncMock()
mock_manager.add_server = mock.AsyncMock()
mock_manager.reload_servers_from_database = mock.AsyncMock()
# Set up test data
@ -195,7 +195,7 @@ async def test_create_mcp_server_direct():
# Verify mocks were called
mock_get_server.assert_called_once_with(mock_prisma, server_id)
mock_create.assert_called_once()
mock_manager.add_update_server.assert_called_once_with(expected_response)
mock_manager.add_server.assert_called_once_with(expected_response)
@pytest.mark.asyncio
@ -379,7 +379,7 @@ async def test_edit_mcp_server_redacts_credentials():
mock_prisma = mock.Mock()
mock_get_prisma.return_value = mock_prisma
mock_manager.add_update_server = mock.AsyncMock()
mock_manager.update_server = mock.AsyncMock()
mock_manager.reload_servers_from_database = mock.AsyncMock()
server_id = str(uuid.uuid4())
@ -417,7 +417,7 @@ async def test_edit_mcp_server_redacts_credentials():
mock_validate.assert_called_once()
mock_update.assert_awaited_once()
mock_manager.add_update_server.assert_called_once_with(updated_server)
mock_manager.update_server.assert_called_once_with(updated_server)
mock_manager.reload_servers_from_database.assert_awaited_once()
def test_validate_mcp_server_name_direct():
"""

127
tests/test_litellm/google_genai/test_google_genai_adapter.py
View file

@ -1197,6 +1197,131 @@ async def test_agenerate_content_x_goog_api_key_header():
# Verify other expected headers
assert headers.get("Content-Type") == "application/json", f"Expected Content-Type application/json, got {headers.get('Content-Type')}"
print(f"✓ Test passed: x-goog-api-key header correctly set to {api_key_value}")
print(f"✓ All headers: {list(headers.keys())}")
def test_inline_data_base64_image_transformation():
"""Test transformation of Gemini inline_data (Base64 images) to OpenAI format"""
from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter
adapter = GoogleGenAIAdapter()
# Test input with Base64 image
model = "gpt-4-vision-preview"
contents = {
"role": "user",
"parts": [
{"text": "What's in this image?"},
{
"inline_data": {
"mime_type": "image/jpeg",
"data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
}
}
]
}
# Transform to completion format
completion_request = adapter.translate_generate_content_to_completion(
model=model,
contents=contents
)
# Verify the transformation
assert completion_request["model"] == model
assert len(completion_request["messages"]) == 1
assert completion_request["messages"][0]["role"] == "user"
# Verify content is an array (multimodal format)
content = completion_request["messages"][0]["content"]
assert isinstance(content, list), "Content should be a list for multimodal messages"
assert len(content) == 2, "Should have 2 content parts (text + image)"
# Verify text part
text_part = content[0]
assert text_part["type"] == "text"
assert text_part["text"] == "What's in this image?"
# Verify image part
image_part = content[1]
assert image_part["type"] == "image_url"
assert "image_url" in image_part
assert "url" in image_part["image_url"]
assert image_part["image_url"]["url"].startswith("data:image/jpeg;base64,")
assert "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" in image_part["image_url"]["url"]
def test_inline_data_image_only_transformation():
"""Test transformation of Gemini inline_data with only image (no text)"""
from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter
adapter = GoogleGenAIAdapter()
# Test input with only Base64 image (no text)
model = "gpt-4-vision-preview"
contents = {
"role": "user",
"parts": [
{
"inline_data": {
"mime_type": "image/png",
"data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
}
}
]
}
# Transform to completion format
completion_request = adapter.translate_generate_content_to_completion(
model=model,
contents=contents
)
# Verify the transformation
assert completion_request["model"] == model
assert len(completion_request["messages"]) == 1
assert completion_request["messages"][0]["role"] == "user"
# Verify content is an array (multimodal format)
content = completion_request["messages"][0]["content"]
assert isinstance(content, list), "Content should be a list for multimodal messages"
assert len(content) == 1, "Should have 1 content part (image only)"
# Verify image part
image_part = content[0]
assert image_part["type"] == "image_url"
assert "image_url" in image_part
assert "url" in image_part["image_url"]
assert image_part["image_url"]["url"].startswith("data:image/png;base64,")
def test_inline_data_backward_compatibility_text_only():
"""Test that pure text messages still use simple string format (backward compatibility)"""
from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter
adapter = GoogleGenAIAdapter()
# Test input with only text (no images)
model = "gpt-3.5-turbo"
contents = {
"role": "user",
"parts": [{"text": "Hello, how are you?"}]
}
# Transform to completion format
completion_request = adapter.translate_generate_content_to_completion(
model=model,
contents=contents
)
# Verify the transformation
assert completion_request["model"] == model
assert len(completion_request["messages"]) == 1
assert completion_request["messages"][0]["role"] == "user"
# Verify content is a simple string (not an array) for backward compatibility
content = completion_request["messages"][0]["content"]
assert isinstance(content, str), "Content should be a string for text-only messages (backward compatibility)"
assert content == "Hello, how are you?"

119
tests/test_litellm/google_genai/test_google_genai_transformation.py
View file

@ -247,3 +247,122 @@ def test_responses_api_no_reasoning():
# reasoning_effort should not be in result if not provided (filtered out as None)
assert "reasoning_effort" not in result or result.get("reasoning_effort") is None
def test_transform_generate_content_request_with_system_instruction():
"""Test that systemInstruction parameter is properly included in the request"""
config = GoogleGenAIConfig()
system_instruction = {
"parts": [{"text": "You are a helpful assistant"}]
}
contents = [
{
"role": "user",
"parts": [{"text": "Hello"}]
}
]
generate_content_config_dict = {
"temperature": 1.0,
"maxOutputTokens": 100
}
# Call transform_generate_content_request
result = config.transform_generate_content_request(
model="gemini-3-flash-preview",
contents=contents,
tools=None,
generate_content_config_dict=generate_content_config_dict,
system_instruction=system_instruction,
)
# Verify that systemInstruction is in the request
assert "systemInstruction" in result, "systemInstruction should be in request body"
assert result["systemInstruction"] == system_instruction, "systemInstruction should match input"
assert result["model"] == "gemini-3-flash-preview"
assert result["contents"] == contents
def test_transform_generate_content_request_without_system_instruction():
"""Test that request works correctly without systemInstruction"""
config = GoogleGenAIConfig()
contents = [
{
"role": "user",
"parts": [{"text": "Hello"}]
}
]
generate_content_config_dict = {
"temperature": 1.0
}
# Call transform_generate_content_request without system_instruction
result = config.transform_generate_content_request(
model="gemini-3-flash-preview",
contents=contents,
tools=None,
generate_content_config_dict=generate_content_config_dict,
system_instruction=None,
)
# Verify that systemInstruction is NOT in the request when not provided
assert "systemInstruction" not in result, "systemInstruction should not be in request when None"
assert result["model"] == "gemini-3-flash-preview"
assert result["contents"] == contents
def test_transform_generate_content_request_system_instruction_with_tools():
"""Test that systemInstruction works correctly alongside tools"""
config = GoogleGenAIConfig()
system_instruction = {
"parts": [{"text": "You are a helpful assistant that uses tools"}]
}
contents = [
{
"role": "user",
"parts": [{"text": "What's the weather?"}]
}
]
tools = [
{
"functionDeclarations": [
{
"name": "get_weather",
"description": "Get weather information",
"parameters": {
"type": "object",
"properties": {
"location": {"type": "string"}
}
}
}
]
}
]
generate_content_config_dict = {
"temperature": 0.7
}
# Call transform_generate_content_request with both system_instruction and tools
result = config.transform_generate_content_request(
model="gemini-3-flash-preview",
contents=contents,
tools=tools,
generate_content_config_dict=generate_content_config_dict,
system_instruction=system_instruction,
)
# Verify that both systemInstruction and tools are in the request
assert "systemInstruction" in result, "systemInstruction should be in request body"
assert result["systemInstruction"] == system_instruction
assert "tools" in result, "tools should be in request body"
assert result["tools"] == tools
assert result["model"] == "gemini-3-flash-preview"

454
tests/test_litellm/integrations/test_opentelemetry.py
View file

@ -8,6 +8,7 @@ from unittest.mock import MagicMock, patch
# Adds the grandparent directory to sys.path to allow importing project modules
sys.path.insert(0, os.path.abspath("../.."))
from opentelemetry import trace
from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
@ -1318,3 +1319,456 @@ class TestOpenTelemetryProtocolSelection(unittest.TestCase):
"http://collector:4317/v1/traces", "logs"
)
self.assertEqual(normalized, "http://collector:4317/v1/logs")
class TestOpenTelemetryExternalSpan(unittest.TestCase):
"""
Test suite for external span handling in OpenTelemetry integration.
These tests verify that LiteLLM correctly handles spans created outside
of LiteLLM (e.g., by Langfuse SDK, user application code, or global context)
without closing them prematurely.
Background:
- External spans can come from: Langfuse SDK, user code, HTTP traceparent headers, global context
- LiteLLM should NEVER close spans it did not create
- Bug: LiteLLM was reusing and closing external spans in _start_primary_span
"""
HERE = os.path.dirname(__file__)
def setUp(self):
"""Set up common test fixtures"""
self.span_exporter = InMemorySpanExporter()
self.tracer_provider = TracerProvider()
self.tracer_provider.add_span_processor(
SimpleSpanProcessor(self.span_exporter)
)
# Don't set global tracer provider - instead, get tracers directly from our provider
# This avoids "Overriding of current TracerProvider is not allowed" warnings
# Clear any existing spans
self.span_exporter.clear()
def _create_test_kwargs_and_response(self):
"""Load test data from JSON files"""
with open(
os.path.join(self.HERE, "open_telemetry", "data", "captured_kwargs.json")
) as f:
kwargs = json.load(f)
with open(
os.path.join(self.HERE, "open_telemetry", "data", "captured_response.json")
) as f:
response_obj = json.load(f)
return kwargs, response_obj
def _get_spans_by_name(self, name):
"""Get all spans with the given name"""
spans = self.span_exporter.get_finished_spans()
return [s for s in spans if s.name == name]
@patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "false"}, clear=False)
def test_external_span_not_closed_with_use_otel_litellm_request_span_false(self):
"""
Test that external spans are not closed when USE_OTEL_LITELLM_REQUEST_SPAN=false (default).
Expected behavior:
- External span remains open (is_recording = True)
- raw_gen_ai_request spans are direct children of external span (shallow hierarchy)
- No litellm_request span is created
- Multiple completions work correctly
"""
# Initialize OpenTelemetry
otel = OpenTelemetry(tracer_provider=self.tracer_provider)
# Load test data
kwargs, response_obj = self._create_test_kwargs_and_response()
# Create external parent span using our test TracerProvider
tracer = self.tracer_provider.get_tracer(__name__)
with tracer.start_as_current_span("external_parent_span") as parent_span:
parent_ctx = parent_span.get_span_context()
parent_trace_id = parent_ctx.trace_id
parent_span_id = parent_ctx.span_id
self.assertTrue(
parent_span.is_recording(),
"External span should be recording before completion calls"
)
# First completion call
start_time = datetime.utcnow()
end_time = start_time + timedelta(seconds=1)
otel._handle_success(kwargs, response_obj, start_time, end_time)
# Verify parent span is still recording
self.assertTrue(
parent_span.is_recording(),
"External span should still be recording after first completion"
)
# Second completion call
start_time2 = end_time
end_time2 = start_time2 + timedelta(seconds=1)
otel._handle_success(kwargs, response_obj, start_time2, end_time2)
# Verify parent span is still recording
self.assertTrue(
parent_span.is_recording(),
"External span should still be recording after second completion"
)
# After exiting context, verify spans
spans = self.span_exporter.get_finished_spans()
# All spans should have the same trace_id
for span in spans:
self.assertEqual(
span.context.trace_id,
parent_trace_id,
f"Span {span.name} should have same trace_id as parent"
)
# Should have external_parent_span
parent_spans = self._get_spans_by_name("external_parent_span")
self.assertEqual(len(parent_spans), 1, "Should have exactly one external_parent_span")
# Verify LiteLLM set attributes on external parent span
parent_span_finished = parent_spans[0]
self.assertIsNotNone(
parent_span_finished.attributes,
"Parent span should have attributes set by LiteLLM"
)
self.assertIn(
"gen_ai.request.model",
parent_span_finished.attributes,
"Parent span should have model attribute from LiteLLM"
)
# Should have raw_gen_ai_request spans (if message_logging is on)
raw_spans = self._get_spans_by_name("raw_gen_ai_request")
# Note: May be 0 if message_logging is off, or 2 if on
# Should NOT have litellm_request spans (USE_OTEL_LITELLM_REQUEST_SPAN=false)
litellm_spans = self._get_spans_by_name("litellm_request")
self.assertEqual(
len(litellm_spans),
0,
"Should NOT have litellm_request spans when USE_OTEL_LITELLM_REQUEST_SPAN=false"
)
# Verify raw_gen_ai_request spans are direct children of external span
for raw_span in raw_spans:
self.assertEqual(
raw_span.parent.span_id if raw_span.parent else None,
parent_span_id,
f"raw_gen_ai_request should be direct child of external_parent_span"
)
@patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "true"}, clear=False)
def test_external_span_not_closed_with_use_otel_litellm_request_span_true(self):
"""
Test that external spans are not closed when USE_OTEL_LITELLM_REQUEST_SPAN=true.
Expected behavior:
- External span remains open (is_recording = True)
- litellm_request spans are created as children of external span
- raw_gen_ai_request spans are children of litellm_request spans
- Correct hierarchy: external_parent → litellm_request → raw_gen_ai_request
"""
# Initialize OpenTelemetry
otel = OpenTelemetry(tracer_provider=self.tracer_provider)
# Load test data
kwargs, response_obj = self._create_test_kwargs_and_response()
# Create external parent span using our test TracerProvider
tracer = self.tracer_provider.get_tracer(__name__)
with tracer.start_as_current_span("external_parent_span") as parent_span:
parent_ctx = parent_span.get_span_context()
parent_trace_id = parent_ctx.trace_id
parent_span_id = parent_ctx.span_id
# First completion call
start_time = datetime.utcnow()
end_time = start_time + timedelta(seconds=1)
otel._handle_success(kwargs, response_obj, start_time, end_time)
# Verify parent span is still recording
self.assertTrue(
parent_span.is_recording(),
"External span should still be recording after first completion"
)
# Second completion call
start_time2 = end_time
end_time2 = start_time2 + timedelta(seconds=1)
otel._handle_success(kwargs, response_obj, start_time2, end_time2)
# Verify parent span is still recording
self.assertTrue(
parent_span.is_recording(),
"External span should still be recording after second completion"
)
# After exiting context, verify spans
spans = self.span_exporter.get_finished_spans()
# All spans should have the same trace_id
for span in spans:
self.assertEqual(
span.context.trace_id,
parent_trace_id,
f"Span {span.name} should have same trace_id as parent"
)
# Should have litellm_request spans (USE_OTEL_LITELLM_REQUEST_SPAN=true)
litellm_spans = self._get_spans_by_name("litellm_request")
self.assertEqual(
len(litellm_spans),
2,
"Should have 2 litellm_request spans when USE_OTEL_LITELLM_REQUEST_SPAN=true"
)
# Verify litellm_request spans are children of external span
for litellm_span in litellm_spans:
self.assertEqual(
litellm_span.parent.span_id if litellm_span.parent else None,
parent_span_id,
"litellm_request should be child of external_parent_span"
)
# Verify raw_gen_ai_request spans (if present) are children of litellm_request
raw_spans = self._get_spans_by_name("raw_gen_ai_request")
if raw_spans:
litellm_span_ids = {s.context.span_id for s in litellm_spans}
for raw_span in raw_spans:
self.assertIn(
raw_span.parent.span_id if raw_span.parent else None,
litellm_span_ids,
"raw_gen_ai_request should be child of litellm_request"
)
@patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "false"}, clear=False)
def test_external_span_with_multiple_completions(self):
"""
Test that multiple completion calls work correctly within external span context.
Expected behavior:
- Both completion calls succeed
- All spans belong to the same trace
- External span remains open throughout
- No errors or warnings about "ended span"
"""
# Initialize OpenTelemetry
otel = OpenTelemetry(tracer_provider=self.tracer_provider)
# Load test data
kwargs, response_obj = self._create_test_kwargs_and_response()
# Create external parent span using our test TracerProvider
tracer = self.tracer_provider.get_tracer(__name__)
with tracer.start_as_current_span("external_parent_span") as parent_span:
parent_ctx = parent_span.get_span_context()
parent_trace_id = parent_ctx.trace_id
# Make multiple completion calls
for i in range(3):
start_time = datetime.utcnow()
end_time = start_time + timedelta(seconds=1)
# This should not raise any exceptions
otel._handle_success(kwargs, response_obj, start_time, end_time)
# Verify parent span is still recording after each call
self.assertTrue(
parent_span.is_recording(),
f"External span should still be recording after completion #{i+1}"
)
# Verify all spans have the same trace_id
spans = self.span_exporter.get_finished_spans()
for span in spans:
self.assertEqual(
span.context.trace_id,
parent_trace_id,
f"All spans should belong to the same trace"
)
# Should have the external parent span
parent_spans = self._get_spans_by_name("external_parent_span")
self.assertEqual(len(parent_spans), 1, "Should have exactly one external_parent_span")
# Verify LiteLLM set attributes on external parent span
parent_span_finished = parent_spans[0]
self.assertIn(
"gen_ai.request.model",
parent_span_finished.attributes,
"Parent span should have model attribute from LiteLLM"
)
@patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "false"}, clear=False)
def test_external_span_from_global_context(self):
"""
Test external span detection from global context (Priority 3 in _get_span_context).
This simulates the case where a span is set in the global context
(e.g., by user code or Langfuse SDK) and LiteLLM detects it via
trace.get_current_span().
Expected behavior:
- LiteLLM detects the span from global context
- External span is not closed
- Correct parent-child relationship
"""
# Initialize OpenTelemetry
otel = OpenTelemetry(tracer_provider=self.tracer_provider)
# Load test data
kwargs, response_obj = self._create_test_kwargs_and_response()
# Create external parent span and set it as current using our test TracerProvider
tracer = self.tracer_provider.get_tracer(__name__)
with tracer.start_as_current_span("external_global_span") as parent_span:
parent_ctx = parent_span.get_span_context()
parent_trace_id = parent_ctx.trace_id
# Verify the span is in global context
current_span = trace.get_current_span()
self.assertEqual(current_span, parent_span, "Span should be in global context")
# Make completion call
start_time = datetime.utcnow()
end_time = start_time + timedelta(seconds=1)
otel._handle_success(kwargs, response_obj, start_time, end_time)
# Verify parent span is still recording
self.assertTrue(
parent_span.is_recording(),
"External span from global context should not be closed"
)
# Verify trace structure
spans = self.span_exporter.get_finished_spans()
for span in spans:
self.assertEqual(
span.context.trace_id,
parent_trace_id,
"All spans should have the same trace_id"
)
@patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "false"}, clear=False)
def test_external_span_hierarchy_preserved(self):
"""
Test that span hierarchy is correctly preserved with external parent.
Expected behavior:
- Parent span IDs are correct
- Trace structure matches expected hierarchy
- Span names are correct
"""
# Initialize OpenTelemetry
otel = OpenTelemetry(tracer_provider=self.tracer_provider)
otel.message_logging = True # Enable message logging to get raw_gen_ai_request spans
# Load test data
kwargs, response_obj = self._create_test_kwargs_and_response()
# Create external parent span using our test TracerProvider
tracer = self.tracer_provider.get_tracer(__name__)
with tracer.start_as_current_span("external_parent_span") as parent_span:
parent_span_id = parent_span.get_span_context().span_id
# Make completion call
start_time = datetime.utcnow()
end_time = start_time + timedelta(seconds=1)
otel._handle_success(kwargs, response_obj, start_time, end_time)
# Verify hierarchy
spans = self.span_exporter.get_finished_spans()
# Get spans by name
parent_spans = self._get_spans_by_name("external_parent_span")
raw_spans = self._get_spans_by_name("raw_gen_ai_request")
self.assertEqual(len(parent_spans), 1, "Should have one parent span")
# Verify parent-child relationship
if raw_spans: # If message_logging is on
for raw_span in raw_spans:
self.assertEqual(
raw_span.parent.span_id if raw_span.parent else None,
parent_span_id,
"raw_gen_ai_request should be child of external_parent_span"
)
@patch.dict(os.environ, {"USE_OTEL_LITELLM_REQUEST_SPAN": "false"}, clear=False)
def test_external_span_not_ended_on_failure(self):
"""
Test that external spans are not closed even on failure.
Expected behavior:
- When _handle_failure is called with external span context
- External span remains open (is_recording = True)
- Error span is created correctly
- External span status is NOT changed by LiteLLM
"""
# Initialize OpenTelemetry
otel = OpenTelemetry(tracer_provider=self.tracer_provider)
# Load test data
kwargs, response_obj = self._create_test_kwargs_and_response()
# Create external parent span using our test TracerProvider
tracer = self.tracer_provider.get_tracer(__name__)
with tracer.start_as_current_span("external_parent_span") as parent_span:
parent_ctx = parent_span.get_span_context()
parent_trace_id = parent_ctx.trace_id
# Simulate failure
start_time = datetime.utcnow()
end_time = start_time + timedelta(seconds=1)
# Create error response object
error_response = {"error": "Test error"}
# Call _handle_failure
otel._handle_failure(kwargs, error_response, start_time, end_time)
# Verify parent span is still recording
self.assertTrue(
parent_span.is_recording(),
"External span should still be recording even after failure"
)
# Verify trace structure
spans = self.span_exporter.get_finished_spans()
# All spans should have the same trace_id
for span in spans:
self.assertEqual(
span.context.trace_id,
parent_trace_id,
"All spans should have the same trace_id even on failure"
)
# Should have external_parent_span
parent_spans = self._get_spans_by_name("external_parent_span")
self.assertEqual(len(parent_spans), 1, "Should have exactly one external_parent_span")
# Verify LiteLLM set attributes on external parent span even on failure
parent_span_finished = parent_spans[0]
self.assertIn(
"gen_ai.request.model",
parent_span_finished.attributes,
"Parent span should have model attribute from LiteLLM even on failure"
)

106
tests/test_litellm/integrations/test_prometheus_metric_name_consistency.py Normal file
View file

@ -0,0 +1,106 @@
"""
Unit tests for prometheus metric name consistency
This test ensures that the metric names used when creating Prometheus metrics
match the names defined in DEFINED_PROMETHEUS_METRICS, so that metric filtering
configuration works correctly.
Related issue: https://github.com/BerriAI/litellm/issues/18221
"""
from typing import get_args
import pytest
def test_remaining_requests_metric_name_in_defined_metrics():
"""
Test that litellm_remaining_requests_metric is defined in DEFINED_PROMETHEUS_METRICS.
The metric name should include the _metric suffix to be consistent with the
configuration format users specify in prometheus_metrics_config.
"""
from litellm.types.integrations.prometheus import DEFINED_PROMETHEUS_METRICS
defined_metrics = get_args(DEFINED_PROMETHEUS_METRICS)
assert (
"litellm_remaining_requests_metric" in defined_metrics
), "litellm_remaining_requests_metric should be in DEFINED_PROMETHEUS_METRICS"
def test_remaining_tokens_metric_name_in_defined_metrics():
"""
Test that litellm_remaining_tokens_metric is defined in DEFINED_PROMETHEUS_METRICS.
The metric name should include the _metric suffix to be consistent with the
configuration format users specify in prometheus_metrics_config.
"""
from litellm.types.integrations.prometheus import DEFINED_PROMETHEUS_METRICS
defined_metrics = get_args(DEFINED_PROMETHEUS_METRICS)
assert (
"litellm_remaining_tokens_metric" in defined_metrics
), "litellm_remaining_tokens_metric should be in DEFINED_PROMETHEUS_METRICS"
def test_prometheus_metric_labels_have_remaining_metrics():
"""
Test that PrometheusMetricLabels has label definitions for remaining metrics.
This ensures that the labels can be retrieved when creating the metrics.
"""
from litellm.types.integrations.prometheus import PrometheusMetricLabels
# Test that labels can be retrieved for remaining metrics
remaining_requests_labels = PrometheusMetricLabels.get_labels(
"litellm_remaining_requests_metric"
)
remaining_tokens_labels = PrometheusMetricLabels.get_labels(
"litellm_remaining_tokens_metric"
)
assert isinstance(
remaining_requests_labels, list
), "Labels for litellm_remaining_requests_metric should be a list"
assert isinstance(
remaining_tokens_labels, list
), "Labels for litellm_remaining_tokens_metric should be a list"
# These metrics should have api_provider and api_base labels
assert (
"api_provider" in remaining_requests_labels
), "litellm_remaining_requests_metric should have api_provider label"
assert (
"api_base" in remaining_requests_labels
), "litellm_remaining_requests_metric should have api_base label"
assert (
"api_provider" in remaining_tokens_labels
), "litellm_remaining_tokens_metric should have api_provider label"
assert (
"api_base" in remaining_tokens_labels
), "litellm_remaining_tokens_metric should have api_base label"
def test_all_defined_metrics_have_consistent_naming():
"""
Test that all metrics defined in DEFINED_PROMETHEUS_METRICS follow
a consistent naming convention.
This helps prevent similar inconsistencies in the future.
"""
from litellm.types.integrations.prometheus import DEFINED_PROMETHEUS_METRICS
defined_metrics = get_args(DEFINED_PROMETHEUS_METRICS)
for metric_name in defined_metrics:
# All metrics should start with 'litellm_'
assert metric_name.startswith(
"litellm_"
), f"Metric {metric_name} should start with 'litellm_'"
if __name__ == "__main__":
test_remaining_requests_metric_name_in_defined_metrics()
test_remaining_tokens_metric_name_in_defined_metrics()
test_prometheus_metric_labels_have_remaining_metrics()
test_all_defined_metrics_have_consistent_naming()
print("All prometheus metric name consistency tests passed!")

51
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
View file

@ -809,3 +809,54 @@ def test_bedrock_anthropic_prompt_caching():
assert completion_cost >= 0
assert round(prompt_cost, 3) == 0.111
assert round(completion_cost, 5) == 0.00820
def test_reasoning_tokens_without_text_tokens_gpt5_nano():
"""
Test fix for GitHub issue #18599:
https://github.com/BerriAI/litellm/issues/18599
When OpenAI models (gpt-5-nano, o1, o3) return reasoning_tokens but don't provide
text_tokens, LiteLLM should calculate text_tokens as:
text_tokens = completion_tokens - reasoning_tokens - audio_tokens - image_tokens
This ensures ALL completion tokens are billed, not just reasoning tokens.
"""
model = "gpt-5-nano"
custom_llm_provider = "openai"
# Simulate OpenAI gpt-5-nano response where text_tokens is NOT provided
# completion_tokens: 977 total
# reasoning_tokens: 768
# text_tokens: should be calculated as 977 - 768 = 209
usage = Usage(
prompt_tokens=17,
completion_tokens=977,
total_tokens=994,
completion_tokens_details=CompletionTokensDetailsWrapper(
reasoning_tokens=768,
audio_tokens=0,
# text_tokens NOT provided - this is the key part of the bug
),
)
prompt_cost, completion_cost = generic_cost_per_token(
model=model,
usage=usage,
custom_llm_provider=custom_llm_provider,
)
# gpt-5-nano pricing: $0.05/1M input, $0.40/1M output
expected_prompt_cost = 17 * 0.05 / 1_000_000
expected_completion_cost = 977 * 0.40 / 1_000_000 # ALL tokens, not just reasoning
assert abs(prompt_cost - expected_prompt_cost) < 1e-10, \
f"Prompt cost incorrect: {prompt_cost} vs {expected_prompt_cost}"
assert abs(completion_cost - expected_completion_cost) < 1e-10, \
f"Completion cost incorrect: {completion_cost} vs {expected_completion_cost}"
# Verify it's NOT using only reasoning_tokens (the bug)
wrong_cost = 768 * 0.40 / 1_000_000 # Only reasoning tokens
assert abs(completion_cost - wrong_cost) > 1e-6, \
"Bug detected: Cost calculation is using only reasoning_tokens instead of all completion_tokens!"

51
tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
View file

@ -497,6 +497,57 @@ def test_convert_gemini_messages():
)
def test_convert_gemini_tool_call_result_with_image_url():
"""
Test that image_url content type in tool results is handled correctly for Gemini.
Fixes: https://github.com/BerriAI/litellm/issues/18187
"""
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_gemini_tool_call_result,
)
from litellm.types.llms.openai import ChatCompletionToolMessage
# Test with string image_url format
message_str_format = ChatCompletionToolMessage(
role="tool",
tool_call_id="call_123",
content=[{"type": "image_url", "image_url": "data:image/jpeg;base64,/9j/4AAQ"}],
)
last_message_with_tool_calls = {
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_123",
"type": "function",
"index": 0,
"function": {"name": "get_image", "arguments": "{}"},
}
],
}
result = convert_to_gemini_tool_call_result(
message=message_str_format,
last_message_with_tool_calls=last_message_with_tool_calls,
)
# Should have inline_data for the image
assert isinstance(result, list) and any("inline_data" in p for p in result)
# Test with dict image_url format (OpenAI standard)
message_dict_format = ChatCompletionToolMessage(
role="tool",
tool_call_id="call_456",
content=[{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQ"}}],
)
last_message_with_tool_calls["tool_calls"][0]["id"] = "call_456"
result2 = convert_to_gemini_tool_call_result(
message=message_dict_format,
last_message_with_tool_calls=last_message_with_tool_calls,
)
assert isinstance(result2, list) and any("inline_data" in p for p in result2)
def test_bedrock_tools_unpack_defs():
"""
Test that the unpack_defs method handles nested $ref inside anyOf items correctly

Some files were not shown because too many files have changed in this diff Show more