scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-11 22:51:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 124

[Fix] - Reliability fix OOMs with image url handling#19257

Browse source
This commit is contained in:
Ishaan Jaffer 2026-02-04 19:16:00 -08:00
parent 1df7957811
commit 0458d52add
7 changed files with 1176 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

198
docs/my-website/docs/proxy/config_settings.md
View file

@ -24,73 +24,81 @@ litellm_settings:
turn_off_message_logging: boolean # prevent the messages and responses from being logged to on your callbacks, but request metadata will still be logged. Useful for privacy/compliance when handling sensitive data.
redact_user_api_key_info: boolean # Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging.
langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] # default tags for Langfuse Logging
# Networking settings
request_timeout: 10 # (int) llm requesttimeout in seconds. Raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout
request_timeout: 10 # (int) llm requesttimeout in seconds. Raise Timeout error if call takes longer than 10s. Sets litellm.request_timeout
force_ipv4: boolean # If true, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6 + Anthropic API
set_verbose: boolean # sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION
# Debugging - see debugging docs for more options
# Use `--debug` or `--detailed_debug` CLI flags, or set LITELLM_LOG env var to "INFO", "DEBUG", or "ERROR"
json_logs: boolean # if true, logs will be in json format
# Fallbacks, reliability
default_fallbacks: ["claude-opus"] # set default_fallbacks, in case a specific model group is misconfigured / bad.
content_policy_fallbacks: [{"gpt-3.5-turbo-small": ["claude-opus"]}] # fallbacks for ContentPolicyErrors
context_window_fallbacks: [{"gpt-3.5-turbo-small": ["gpt-3.5-turbo-large", "claude-opus"]}] # fallbacks for ContextWindowExceededErrors
content_policy_fallbacks: [{ "gpt-3.5-turbo-small": ["claude-opus"] }] # fallbacks for ContentPolicyErrors
context_window_fallbacks: [{ "gpt-3.5-turbo-small": ["gpt-3.5-turbo-large", "claude-opus"] }] # fallbacks for ContextWindowExceededErrors
# MCP Aliases - Map aliases to MCP server names for easier tool access
mcp_aliases: { "github": "github_mcp_server", "zapier": "zapier_mcp_server", "deepwiki": "deepwiki_mcp_server" } # Maps friendly aliases to MCP server names. Only the first alias for each server is used
mcp_aliases: {
"github": "github_mcp_server",
"zapier": "zapier_mcp_server",
"deepwiki": "deepwiki_mcp_server",
} # Maps friendly aliases to MCP server names. Only the first alias for each server is used
# Caching settings
cache: true
cache_params: # set cache params for redis
type: redis # type of cache to initialize
cache: true
cache_params: # set cache params for redis
type: redis # type of cache to initialize (options: "local", "redis", "s3", "gcs")
# Optional - Redis Settings
host: "localhost" # The host address for the Redis cache. Required if type is "redis".
port: 6379 # The port number for the Redis cache. Required if type is "redis".
password: "your_password" # The password for the Redis cache. Required if type is "redis".
host: "localhost" # The host address for the Redis cache. Required if type is "redis".
port: 6379 # The port number for the Redis cache. Required if type is "redis".
password: "your_password" # The password for the Redis cache. Required if type is "redis".
namespace: "litellm.caching.caching" # namespace for redis cache
max_connections: 100 # [OPTIONAL] Set Maximum number of Redis connections. Passed directly to redis-py.
# Optional - Redis Cluster Settings
redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}]
redis_startup_nodes: [{ "host": "127.0.0.1", "port": "7001" }]
# Optional - Redis Sentinel Settings
service_name: "mymaster"
sentinel_nodes: [["localhost", 26379]]
# Optional - GCP IAM Authentication for Redis
gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" # GCP service account for IAM authentication
gcp_ssl_ca_certs: "./server-ca.pem" # Path to SSL CA certificate file for GCP Memorystore Redis
ssl: true # Enable SSL for secure connections
ssl_cert_reqs: null # Set to null for self-signed certificates
ssl_check_hostname: false # Set to false for self-signed certificates
gcp_service_account: "projects/-/serviceAccounts/your-sa@project.iam.gserviceaccount.com" # GCP service account for IAM authentication
gcp_ssl_ca_certs: "./server-ca.pem" # Path to SSL CA certificate file for GCP Memorystore Redis
ssl: true # Enable SSL for secure connections
ssl_cert_reqs: null # Set to null for self-signed certificates
ssl_check_hostname: false # Set to false for self-signed certificates
# Optional - Qdrant Semantic Cache Settings
qdrant_semantic_cache_embedding_model: openai-embedding # the model should be defined on the model_list
qdrant_collection_name: test_collection
qdrant_quantization_config: binary
similarity_threshold: 0.8 # similarity threshold for semantic cache
similarity_threshold: 0.8 # similarity threshold for semantic cache
# Optional - S3 Cache Settings
s3_bucket_name: cache-bucket-litellm # AWS Bucket Name for S3
s3_region_name: us-west-2 # AWS Region Name for S3
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 bucket
s3_bucket_name: cache-bucket-litellm # AWS Bucket Name for S3
s3_region_name: us-west-2 # AWS Region Name for S3
s3_aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # us os.environ/<variable name> to pass environment variables. This is AWS Access Key ID for S3
s3_aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY # AWS Secret Access Key for S3
s3_endpoint_url: https://s3.amazonaws.com # [OPTIONAL] S3 endpoint URL, if you want to use Backblaze/cloudflare s3 bucket
# Optional - GCS Cache Settings
gcs_bucket_name: cache-bucket-litellm # GCS Bucket Name for caching
gcs_path_service_account: os.environ/GCS_PATH_SERVICE_ACCOUNT # Path to GCS service account JSON file
gcs_path: cache/ # [OPTIONAL] GCS path prefix for cache objects
# Common Cache settings
# Optional - Supported call types for caching
supported_call_types: ["acompletion", "atext_completion", "aembedding", "atranscription"]
# /chat/completions, /completions, /embeddings, /audio/transcriptions
supported_call_types:
["acompletion", "atext_completion", "aembedding", "atranscription"]
# /chat/completions, /completions, /embeddings, /audio/transcriptions
mode: default_off # if default_off, you need to opt in to caching on a per call basis
ttl: 600 # ttl for caching
disable_copilot_system_to_assistant: False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
disable_copilot_system_to_assistant: False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
callback_settings:
otel:
message_logging: boolean # OTEL logging callback specific settings
message_logging: boolean # OTEL logging callback specific settings
general_settings:
completion_model: string
@ -104,21 +112,23 @@ general_settings:
disable_responses_id_security: boolean # turn off response ID security checks that prevent users from accessing other users' responses
enable_jwt_auth: boolean # allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims
enforce_user_param: boolean # requires all openai endpoint requests to have a 'user' param
reject_clientside_metadata_tags: boolean # if true, rejects requests with client-side 'metadata.tags' to prevent users from influencing budgets
allowed_routes: ["route1", "route2"] # list of allowed proxy API routes - a user can access. (currently JWT-Auth only)
key_management_system: google_kms # either google_kms or azure_kms
master_key: string
maximum_spend_logs_retention_period: 30d # The maximum time to retain spend logs before deletion.
maximum_spend_logs_retention_interval: 1d # interval in which the spend log cleanup task should run in.
user_mcp_management_mode: restricted # or "view_all"
# Database Settings
database_url: string
database_connection_pool_limit: 0 # default 100
database_connection_pool_limit: 0 # default 10
database_connection_timeout: 0 # default 60s
allow_requests_on_db_unavailable: boolean # if true, will allow requests that can not connect to the DB to verify Virtual Key to still work
custom_auth: string
max_parallel_requests: 0 # the max parallel requests allowed per deployment
global_max_parallel_requests: 0 # the max parallel requests allowed on the proxy all up
max_parallel_requests: 0 # the max parallel requests allowed per deployment
global_max_parallel_requests: 0 # the max parallel requests allowed on the proxy all up
infer_model_from_keys: true
background_health_checks: true
health_check_interval: 300
@ -136,6 +146,7 @@ router_settings:
cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
disable_cooldowns: True # bool - Disable cooldowns for all models
enable_tag_filtering: True # bool - Use tag based routing for requests
tag_filtering_match_any: True # bool - Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags
retry_policy: { # Dict[str, int]: retry policy for different types of exceptions
"AuthenticationErrorRetries": 3,
"TimeoutErrorRetries": 3,
@ -170,7 +181,7 @@ router_settings:
| redact_user_api_key_info | boolean | If true, redacts information about the user api key from logs [Proxy Logging](logging#redacting-userapikeyinfo) |
| mcp_aliases | object | Maps friendly aliases to MCP server names for easier tool access. Only the first alias for each server is used. [MCP Aliases](../mcp#mcp-aliases) |
| langfuse_default_tags | array of strings | Default tags for Langfuse Logging. Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields as tags. [Further docs](./logging#litellm-specific-tags-on-langfuse---cache_hit-cache_key) |
| set_verbose | boolean | If true, sets litellm.set_verbose=True to view verbose debug logs. DO NOT LEAVE THIS ON IN PRODUCTION |
| set_verbose | boolean | [DEPRECATED - see debugging docs](./debugging) Use `--debug` or `--detailed_debug` CLI flags, or set `LITELLM_LOG` env var to "INFO", "DEBUG", or "ERROR" instead. |
| json_logs | boolean | If true, logs will be in json format. If you need to store the logs as JSON, just set the `litellm.json_logs = True`. We currently just log the raw POST request from litellm as a JSON [Further docs](./debugging) |
| default_fallbacks | array of strings | List of fallback models to use if a specific model group is misconfigured / bad. [Further docs](./reliability#default-fallbacks) |
| request_timeout | integer | The timeout for requests in seconds. If not set, the default value is `6000 seconds`. [For reference OpenAI Python SDK defaults to `600 seconds`.](https://github.com/openai/openai-python/blob/main/src/openai/_constants.py) |
@ -201,6 +212,7 @@ router_settings:
| disable_responses_id_security | boolean | If true, disables response ID security checks that prevent users from accessing response IDs from other users. When false (default), response IDs are encrypted with user information to ensure users can only access their own responses. Applies to /v1/responses endpoints |
| enable_jwt_auth | boolean | allow proxy admin to auth in via jwt tokens with 'litellm_proxy_admin' in claims. [Doc on JWT Tokens](token_auth) |
| enforce_user_param | boolean | If true, requires all OpenAI endpoint requests to have a 'user' param. [Doc on call hooks](call_hooks)|
| reject_clientside_metadata_tags | boolean | If true, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata. |
| allowed_routes | array of strings | List of allowed proxy API routes a user can access [Doc on controlling allowed routes](enterprise#control-available-public-private-routes)|
| key_management_system | string | Specifies the key management system. [Doc Secret Managers](../secret) |
| master_key | string | The master key for the proxy [Set up Virtual Keys](virtual_keys) |
@ -227,12 +239,13 @@ router_settings:
| image_generation_model | str | The default model to use for image generation - ignores model set in request |
| store_model_in_db | boolean | If true, enables storing model + credential information in the DB. |
| supported_db_objects | List[str] | Fine-grained control over which object types to load from the database when `store_model_in_db` is True. Available types: `"models"`, `"mcp"`, `"guardrails"`, `"vector_stores"`, `"pass_through_endpoints"`, `"prompts"`, `"model_cost_map"`. If not set, all object types are loaded (default behavior). Example: `supported_db_objects: ["mcp"]` to only load MCP servers from DB. |
| user_mcp_management_mode | string | Controls what non-admins can see on the MCP dashboard. `restricted` (default) only lists MCP servers that the user’s teams are explicitly allowed to access. `view_all` lets every user see the full MCP server list. Tool list/call always respects per-key permissions, so users still cannot run MCP calls without access. |
| store_prompts_in_spend_logs | boolean | If true, allows prompts and responses to be stored in the spend logs table. |
| max_request_size_mb | int | The maximum size for requests in MB. Requests above this size will be rejected. |
| max_response_size_mb | int | The maximum size for responses in MB. LLM Responses above this size will not be sent. |
| proxy_budget_rescheduler_min_time | int | The minimum time (in seconds) to wait before checking db for budget resets. **Default is 597 seconds** |
| proxy_budget_rescheduler_max_time | int | The maximum time (in seconds) to wait before checking db for budget resets. **Default is 605 seconds** |
| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 30 seconds** |
| proxy_batch_write_at | int | Time (in seconds) to wait before batch writing spend logs to the db. **Default is 10 seconds** |
| proxy_batch_polling_interval | int | Time (in seconds) to wait before polling a batch, to check if it's completed. **Default is 6000 seconds (1 hour)** |
| alerting_args | dict | Args for Slack Alerting [Doc on Slack Alerting](./alerting.md) |
| custom_key_generate | str | Custom function for key generation [Doc on custom key generation](./virtual_keys.md#custom--key-generate) |
@ -261,13 +274,14 @@ router_settings:
| forward_openai_org_id | boolean | If true, forwards the OpenAI Organization ID to the backend LLM call (if it's OpenAI). |
| forward_client_headers_to_llm_api | boolean | If true, forwards the client headers (any `x-` headers and `anthropic-beta` headers) to the backend LLM call |
| maximum_spend_logs_retention_period | str | Used to set the max retention time for spend logs in the db, after which they will be auto-purged |
| maximum_spend_logs_retention_interval | str | Used to set the interval in which the spend log cleanup task should run in. |
| maximum_spend_logs_retention_interval | str | Used to set the interval in which the spend log cleanup task should run in. |
### router_settings - Reference
:::info
Most values can also be set via `litellm_settings`. If you see overlapping values, settings on `router_settings` will override those on `litellm_settings`.
:::
Most values can also be set via `litellm_settings`. If you see overlapping values, settings on
`router_settings` will override those on `litellm_settings`. :::
```yaml
router_settings:
@ -275,11 +289,12 @@ router_settings:
redis_host: <your-redis-host> # string
redis_password: <your-redis-password> # string
redis_port: <your-redis-port> # string
enable_pre_call_checks: true # bool - Before call is made check if a call is within model context window
allowed_fails: 3 # cooldown model if it fails > 1 call in a minute.
enable_pre_call_checks: true # bool - Before call is made check if a call is within model context window
allowed_fails: 3 # cooldown model if it fails > 1 call in a minute.
cooldown_time: 30 # (in seconds) how long to cooldown model if fails/min > allowed_fails
disable_cooldowns: True # bool - Disable cooldowns for all models
disable_cooldowns: True # bool - Disable cooldowns for all models
enable_tag_filtering: True # bool - Use tag based routing for requests
tag_filtering_match_any: True # bool - Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags
retry_policy: { # Dict[str, int]: retry policy for different types of exceptions
"AuthenticationErrorRetries": 3,
"TimeoutErrorRetries": 3,
@ -289,11 +304,11 @@ router_settings:
}
allowed_fails_policy: {
"BadRequestErrorAllowedFails": 1000, # Allow 1000 BadRequestErrors before cooling down a deployment
"AuthenticationErrorAllowedFails": 10, # int
"TimeoutErrorAllowedFails": 12, # int
"RateLimitErrorAllowedFails": 10000, # int
"ContentPolicyViolationErrorAllowedFails": 15, # int
"InternalServerErrorAllowedFails": 20, # int
"AuthenticationErrorAllowedFails": 10, # int
"TimeoutErrorAllowedFails": 12, # int
"RateLimitErrorAllowedFails": 10000, # int
"ContentPolicyViolationErrorAllowedFails": 15, # int
"InternalServerErrorAllowedFails": 20, # int
}
content_policy_fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for content policy violations
fallbacks=[{"claude-2": ["my-fallback-model"]}] # List[Dict[str, List[str]]]: Fallback model for all errors
@ -309,6 +324,7 @@ router_settings:
| content_policy_fallbacks | array of objects | Specifies fallback models for content policy violations. [More information here](reliability) |
| fallbacks | array of objects | Specifies fallback models for all types of errors. [More information here](reliability) |
| enable_tag_filtering | boolean | If true, uses tag based routing for requests [Tag Based Routing](tag_routing) |
| tag_filtering_match_any | boolean | Tag matching behavior (only when enable_tag_filtering=true). `true`: match if deployment has ANY requested tag; `false`: match only if deployment has ALL requested tags |
| cooldown_time | integer | The duration (in seconds) to cooldown a model if it exceeds the allowed failures. |
| disable_cooldowns | boolean | If true, disables cooldowns for all models. [More information here](reliability) |
| retry_policy | object | Specifies the number of retries for different types of exceptions. [More information here](reliability) |
@ -331,7 +347,7 @@ router_settings:
| caching_groups | Optional[List[tuple]] | List of model groups for caching across model groups. Defaults to None. - e.g. caching_groups=[("openai-gpt-3.5-turbo", "azure-gpt-3.5-turbo")]|
| alerting_config | AlertingConfig | [SDK-only arg] Slack alerting configuration. Defaults to None. [Further Docs](../routing.md#alerting-) |
| assistants_config | AssistantsConfig | Set on proxy via `assistant_settings`. [Further docs](../assistants.md) |
| set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging.md) If true, sets the logging level to verbose. |
| set_verbose | boolean | [DEPRECATED PARAM - see debug docs](./debugging) If true, sets the logging level to verbose. |
| retry_after | int | Time to wait before retrying a request in seconds. Defaults to 0. If `x-retry-after` is received from LLM API, this value is overridden. |
| provider_budget_config | ProviderBudgetConfig | Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None. [Further Docs](./provider_budget_routing.md) |
| enable_pre_call_checks | boolean | If true, checks if a call is within the model's context window before making the call. [More information here](reliability) |
@ -343,6 +359,7 @@ router_settings:
| optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Currently supported: 'router_budget_limiting', 'prompt_caching' |
| ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. |
| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) |
| guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) |
### environment variables - Reference
@ -357,6 +374,7 @@ router_settings:
| AISPEND_ACCOUNT_ID | Account ID for AI Spend
| AISPEND_API_KEY | API Key for AI Spend
| AIOHTTP_CONNECTOR_LIMIT | Connection limit for aiohttp connector. When set to 0, no limit is applied. **Default is 0**
| AIOHTTP_CONNECTOR_LIMIT_PER_HOST | Connection limit per host for aiohttp connector. When set to 0, no limit is applied. **Default is 0**
| AIOHTTP_KEEPALIVE_TIMEOUT | Keep-alive timeout for aiohttp connections in seconds. **Default is 120**
| AIOHTTP_TRUST_ENV | Flag to enable aiohttp trust environment. When this is set to True, aiohttp will respect HTTP(S)_PROXY env vars. **Default is False**
| AIOHTTP_TTL_DNS_CACHE | DNS cache time-to-live for aiohttp in seconds. **Default is 300**
@ -375,6 +393,8 @@ router_settings:
| ATHINA_API_KEY | API key for Athina service
| ATHINA_BASE_URL | Base URL for Athina service (defaults to `https://log.athina.ai`)
| AUTH_STRATEGY | Strategy used for authentication (e.g., OAuth, API key)
| AUTO_REDIRECT_UI_LOGIN_TO_SSO | Flag to enable automatic redirect of UI login page to SSO when SSO is configured. Default is **true**
| AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024
| ANTHROPIC_API_KEY | API key for Anthropic service
| ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
| AWS_ACCESS_KEY_ID | Access Key ID for AWS services
@ -407,6 +427,12 @@ router_settings:
| AZURE_FEDERATED_TOKEN_FILE | File path to Azure federated token
| AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY | Cost per GB per day for Azure File Search service
| AZURE_SCOPE | For EntraID Auth, Scope for Azure services, defaults to "https://cognitiveservices.azure.com/.default"
| AZURE_SENTINEL_DCR_IMMUTABLE_ID | Immutable ID of the Data Collection Rule for Azure Sentinel logging
| AZURE_SENTINEL_STREAM_NAME | Stream name for Azure Sentinel logging
| AZURE_SENTINEL_CLIENT_SECRET | Client secret for Azure Sentinel authentication
| AZURE_SENTINEL_ENDPOINT | Endpoint for Azure Sentinel logging
| AZURE_SENTINEL_TENANT_ID | Tenant ID for Azure Sentinel authentication
| AZURE_SENTINEL_CLIENT_ID | Client ID for Azure Sentinel authentication
| AZURE_KEY_VAULT_URI | URI for Azure Key Vault
| AZURE_OPERATION_POLLING_TIMEOUT | Timeout in seconds for Azure operation polling
| AZURE_STORAGE_ACCOUNT_KEY | The Azure Storage Account Key to use for Authentication to Azure Blob Storage logging
@ -437,6 +463,7 @@ router_settings:
| CYBERARK_CLIENT_CERT | Path to client certificate for CyberArk authentication
| CYBERARK_CLIENT_KEY | Path to client key for CyberArk authentication
| CYBERARK_USERNAME | Username for CyberArk authentication
| CYBERARK_SSL_VERIFY | Flag to enable or disable SSL certificate verification for CyberArk. Default is True
| CONFIDENT_API_KEY | API key for DeepEval integration
| CUSTOM_TIKTOKEN_CACHE_DIR | Custom directory for Tiktoken cache
| CONFIDENT_API_KEY | API key for Confident AI (Deepeval) Logging service
@ -450,6 +477,9 @@ router_settings:
| DATABASE_USER | Username for database connection
| DATABASE_USERNAME | Alias for database user
| DATABRICKS_API_BASE | Base URL for Databricks API
| DATABRICKS_CLIENT_ID | Client ID for Databricks OAuth M2M authentication (Service Principal application ID)
| DATABRICKS_CLIENT_SECRET | Client secret for Databricks OAuth M2M authentication
| DATABRICKS_USER_AGENT | Custom user agent string for Databricks API requests. Used for partner telemetry attribution
| DAYS_IN_A_MONTH | Days in a month for calculation purposes. Default is 28
| DAYS_IN_A_WEEK | Days in a week for calculation purposes. Default is 7
| DAYS_IN_A_YEAR | Days in a year for calculation purposes. Default is 365
@ -471,13 +501,17 @@ router_settings:
| DD_VERSION | Version identifier for Datadog logs. Defaults to "unknown"
| DEBUG_OTEL | Enable debug mode for OpenTelemetry
| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3
| DEFAULT_A2A_AGENT_TIMEOUT | Default timeout in seconds for A2A (Agent-to-Agent) protocol requests. Default is 6000
| DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096
| DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512
| DEFAULT_CHUNK_OVERLAP | Default chunk overlap for RAG text splitters. Default is 200
| DEFAULT_CHUNK_SIZE | Default chunk size for RAG text splitters. Default is 1000
| DEFAULT_CLIENT_DISCONNECT_CHECK_TIMEOUT_SECONDS | Timeout in seconds for checking client disconnection. Default is 1
| DEFAULT_COOLDOWN_TIME_SECONDS | Duration in seconds to cooldown a model after failures. Default is 5
| DEFAULT_CRON_JOB_LOCK_TTL_SECONDS | Time-to-live for cron job locks in seconds. Default is 60 (1 minute)
| DEFAULT_DATAFORSEO_LOCATION_CODE | Default location code for DataForSEO search API. Default is 2250 (France)
| DEFAULT_FAILURE_THRESHOLD_PERCENT | Threshold percentage of failures to cool down a deployment. Default is 0.5 (50%)
| DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS | Minimum number of requests before applying error rate cooldown. Prevents cooldown from triggering on first failure. Default is 5
| DEFAULT_FLUSH_INTERVAL_SECONDS | Default interval in seconds for flushing operations. Default is 5
| DEFAULT_HEALTH_CHECK_INTERVAL | Default interval in seconds for health checks. Default is 300 (5 minutes)
| DEFAULT_HEALTH_CHECK_PROMPT | Default prompt used during health checks for non-image models. Default is "test from litellm"
@ -531,10 +565,14 @@ router_settings:
| DOCS_TITLE | Title of the documentation pages
| DOCS_URL | The path to the Swagger API documentation. **By default this is "/"**
| EMAIL_LOGO_URL | URL for the logo used in emails
| EMAIL_BUDGET_ALERT_TTL | Time-to-live for email budget alerts in seconds
| EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE | Maximum spend percentage for triggering email budget alerts
| EMAIL_SUPPORT_CONTACT | Support contact email address
| EMAIL_SIGNATURE | Custom HTML footer/signature for all emails. Can include HTML tags for formatting and links.
| EMAIL_SUBJECT_INVITATION | Custom subject template for invitation emails.
| EMAIL_SUBJECT_KEY_CREATED | Custom subject template for key creation emails.
| EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE | Percentage of max budget that triggers alerts (as decimal: 0.8 = 80%). Default is 0.8
| EMAIL_BUDGET_ALERT_TTL | Time-to-live for budget alert deduplication in seconds. Default is 86400 (24 hours)
| ENKRYPTAI_API_BASE | Base URL for EnkryptAI Guardrails API. **Default is https://api.enkryptai.com**
| ENKRYPTAI_API_KEY | API key for EnkryptAI Guardrails service
| EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False**
@ -543,6 +581,18 @@ router_settings:
| FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56
| FIREWORKS_AI_80_B | Size parameter for Fireworks AI 80B model. Default is 80
| FIREWORKS_AI_176_B_MOE | Size parameter for Fireworks AI 176B MOE model. Default is 176
| FOCUS_PROVIDER | Destination provider for Focus exports (e.g., `s3`). Defaults to `s3`.
| FOCUS_FORMAT | Output format for Focus exports. Defaults to `parquet`.
| FOCUS_FREQUENCY | Frequency for scheduled Focus exports (`hourly`, `daily`, or `interval`). Defaults to `hourly`.
| FOCUS_CRON_OFFSET | Minute offset used when scheduling hourly/daily Focus exports. Defaults to `5` minutes.
| FOCUS_INTERVAL_SECONDS | Interval (in seconds) for Focus exports when `frequency` is `interval`.
| FOCUS_PREFIX | Object key prefix (or folder) used when uploading Focus export files. Defaults to `focus_exports`.
| FOCUS_S3_BUCKET_NAME | S3 bucket to upload Focus export files when using the S3 destination.
| FOCUS_S3_REGION_NAME | AWS region for the Focus export S3 bucket.
| FOCUS_S3_ENDPOINT_URL | Custom endpoint for the Focus export S3 client (optional; useful for S3-compatible storage).
| FOCUS_S3_ACCESS_KEY | AWS access key ID used by the Focus export S3 client.
| FOCUS_S3_SECRET_KEY | AWS secret access key used by the Focus export S3 client.
| FOCUS_S3_SESSION_TOKEN | AWS session token used by the Focus export S3 client (optional).
| FUNCTION_DEFINITION_TOKEN_COUNT | Token count for function definitions. Default is 9
| GALILEO_BASE_URL | Base URL for Galileo platform
| GALILEO_PASSWORD | Password for Galileo authentication
@ -572,6 +622,8 @@ router_settings:
| GENERIC_USER_PROVIDER_ATTRIBUTE | Attribute specifying the user's provider
| GENERIC_USER_ROLE_ATTRIBUTE | Attribute specifying the user's role
| GENERIC_USERINFO_ENDPOINT | Endpoint to fetch user information in generic OAuth
| GENERIC_LOGGER_ENDPOINT | Endpoint URL for the Generic Logger callback to send logs to
| GENERIC_LOGGER_HEADERS | JSON string of headers to include in Generic Logger callback requests
| GEMINI_API_BASE | Base URL for Gemini API. Default is https://generativelanguage.googleapis.com
| GALILEO_BASE_URL | Base URL for Galileo platform
| GALILEO_PASSWORD | Password for Galileo authentication
@ -584,6 +636,8 @@ router_settings:
| GREENSCALE_ENDPOINT | Endpoint URL for Greenscale service
| GRAYSWAN_API_BASE | Base URL for GraySwan API. Default is https://api.grayswan.ai
| GRAYSWAN_API_KEY | API key for GraySwan Cygnal service
| GRAYSWAN_REASONING_MODE | Reasoning mode for GraySwan guardrail
| GRAYSWAN_VIOLATION_THRESHOLD | Violation threshold for GraySwan guardrail
| GOOGLE_APPLICATION_CREDENTIALS | Path to Google Cloud credentials JSON file
| GOOGLE_CLIENT_ID | Client ID for Google OAuth
| GOOGLE_CLIENT_SECRET | Client secret for Google OAuth
@ -608,6 +662,10 @@ router_settings:
| HELICONE_API_BASE | Base URL for Helicone service, defaults to `https://api.helicone.ai`
| HOSTNAME | Hostname for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog)
| HOURS_IN_A_DAY | Hours in a day for calculation purposes. Default is 24
| HIDDENLAYER_API_BASE | Base URL for HiddenLayer API. Defaults to `https://api.hiddenlayer.ai`
| HIDDENLAYER_AUTH_URL | Authentication URL for HiddenLayer. Defaults to `https://auth.hiddenlayer.ai`
| HIDDENLAYER_CLIENT_ID | Client ID for HiddenLayer SaaS authentication
| HIDDENLAYER_CLIENT_SECRET | Client secret for HiddenLayer SaaS authentication
| HUGGINGFACE_API_BASE | Base URL for Hugging Face API
| HUGGINGFACE_API_KEY | API key for Hugging Face API
| HUMANLOOP_PROMPT_CACHE_TTL_SECONDS | Time-to-live in seconds for cached prompts in Humanloop. Default is 60
@ -630,12 +688,14 @@ router_settings:
| LANGFUSE_PUBLIC_KEY | Public key for Langfuse authentication
| LANGFUSE_RELEASE | Release version of Langfuse integration
| LANGFUSE_SECRET_KEY | Secret key for Langfuse authentication
| LANGFUSE_PROPAGATE_TRACE_ID | Flag to enable propagating trace ID to Langfuse. Default is False
| LANGSMITH_API_KEY | API key for Langsmith platform
| LANGSMITH_BASE_URL | Base URL for Langsmith service
| LANGSMITH_BATCH_SIZE | Batch size for operations in Langsmith
| LANGSMITH_DEFAULT_RUN_NAME | Default name for Langsmith run
| LANGSMITH_PROJECT | Project name for Langsmith integration
| LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging
| LANGSMITH_TENANT_ID | Tenant ID for Langsmith multi-tenant deployments
| LANGTRACE_API_KEY | API key for Langtrace service
| LASSO_API_BASE | Base URL for Lasso API
| LASSO_API_KEY | API key for Lasso service
@ -647,14 +707,18 @@ router_settings:
| LITERAL_API_URL | API URL for Literal service
| LITERAL_BATCH_SIZE | Batch size for Literal operations
| LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX | Disable automatic URL suffix appending for Anthropic API base URLs. When set to `true`, prevents LiteLLM from automatically adding `/v1/messages` or `/v1/complete` to custom Anthropic API endpoints
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
| LITELLM_DD_AGENT_PORT | Port of DataDog agent for LiteLLM-specific log intake. Default is 10518
| LITELLM_DONT_SHOW_FEEDBACK_BOX | Flag to hide feedback box in LiteLLM UI
| LITELLM_DROP_PARAMS | Parameters to drop in LiteLLM requests
| LITELLM_MODIFY_PARAMS | Parameters to modify in LiteLLM requests
| LITELLM_EMAIL | Email associated with LiteLLM account
| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES | Maximum retries for parallel requests in LiteLLM
| LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT | Timeout for retries of parallel requests in LiteLLM
| LITELLM_DISABLE_LAZY_LOADING | When set to "1", "true", "yes", or "on", disables lazy loading of attributes (currently only affects encoding/tiktoken). This ensures encoding is initialized before VCR starts recording HTTP requests, fixing VCR cassette creation issues. See [issue #18659](https://github.com/BerriAI/litellm/issues/18659)
| LITELLM_MIGRATION_DIR | Custom migrations directory for prisma migrations, used for baselining db in read-only file systems.
| LITELLM_HOSTED_UI | URL of the hosted UI for LiteLLM
| LITELLM_UI_API_DOC_BASE_URL | Optional override for the API Reference base URL (used in sample code/docs) when the admin UI runs on a different host than the proxy. Defaults to `PROXY_BASE_URL` when unset.
| LITELM_ENVIRONMENT | Environment of LiteLLM Instance, used by logging services. Currently only used by DeepEval.
| LITELLM_KEY_ROTATION_ENABLED | Enable auto-key rotation for LiteLLM (boolean). Default is false.
| LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS | Interval in seconds for how often to run job that auto-rotates keys. Default is 86400 (24 hours).
@ -671,16 +735,27 @@ router_settings:
| LITELLM_MODE | Operating mode for LiteLLM (e.g., production, development)
| LITELLM_NON_ROOT | Flag to run LiteLLM in non-root mode for enhanced security in Docker containers
| LITELLM_RATE_LIMIT_WINDOW_SIZE | Rate limit window size for LiteLLM. Default is 60
| LITELLM_REASONING_AUTO_SUMMARY | If set to "true", automatically enables detailed reasoning summaries for reasoning models (e.g., o1, o3-mini, deepseek-reasoner). When enabled, adds `summary: "detailed"` to reasoning effort configurations. Default is "false"
| LITELLM_SALT_KEY | Salt key for encryption in LiteLLM
| LITELLM_SSL_CIPHERS | SSL/TLS cipher configuration for faster handshakes. Controls cipher suite preferences for OpenSSL connections.
| LITELLM_SECRET_AWS_KMS_LITELLM_LICENSE | AWS KMS encrypted license for LiteLLM
| LITELLM_TOKEN | Access token for LiteLLM integration
| LITELLM_USER_AGENT | Custom user agent string for LiteLLM API requests. Used for partner telemetry attribution
| LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging
| LITELM_ENVIRONMENT | Environment for LiteLLM Instance. This is currently only logged to DeepEval to determine the environment for DeepEval integration.
| LOGFIRE_TOKEN | Token for Logfire logging service
| LOGFIRE_BASE_URL | Base URL for Logfire logging service (useful for self hosted deployments)
| LOGGING_WORKER_CONCURRENCY | Maximum number of concurrent coroutine slots for the logging worker on the asyncio event loop. Default is 100. Setting too high will flood the event loop with logging tasks which will lower the overall latency of the requests.
| LOGGING_WORKER_MAX_QUEUE_SIZE | Maximum size of the logging worker queue. When the queue is full, the worker aggressively clears tasks to make room instead of dropping logs. Default is 50,000
| LOGGING_WORKER_MAX_TIME_PER_COROUTINE | Maximum time in seconds allowed for each coroutine in the logging worker before timing out. Default is 20.0
| LOGGING_WORKER_CLEAR_PERCENTAGE | Percentage of the queue to extract when clearing. Default is 50%
| MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000
| MAX_ITERATIONS_TO_CLEAR_QUEUE | Maximum number of iterations to attempt when clearing the logging worker queue during shutdown. Default is 200
| MAX_TIME_TO_CLEAR_QUEUE | Maximum time in seconds to spend clearing the logging worker queue during shutdown. Default is 5.0
| LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS | Cooldown time in seconds before allowing another aggressive clear operation when the queue is full. Default is 0.5
| MAX_STRING_LENGTH_PROMPT_IN_DB | Maximum length for strings in spend logs when sanitizing request bodies. Strings longer than this will be truncated. Default is 1000
| MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000
| MAX_IMAGE_URL_DOWNLOAD_SIZE_MB | Maximum size in MB for downloading images from URLs. Prevents memory issues from downloading very large images. Images exceeding this limit will be rejected before download. Set to 0 to completely disable image URL handling (all image_url requests will be blocked). Default is 50MB (matching [OpenAI's limit](https://platform.openai.com/docs/guides/images-vision?api-mode=chat#image-input-requirements))
| MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the long side of high-resolution images. Default is 2000
| MAX_REDIS_BUFFER_DEQUEUE_COUNT | Maximum count for Redis buffer dequeue operations. Default is 100
| MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the short side of high-resolution images. Default is 768
@ -698,10 +773,18 @@ router_settings:
| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024
| MISTRAL_API_BASE | Base URL for Mistral API. Default is https://api.mistral.ai
| MISTRAL_API_KEY | API key for Mistral API
| MICROSOFT_AUTHORIZATION_ENDPOINT | Custom authorization endpoint URL for Microsoft SSO (overrides default Microsoft OAuth authorization endpoint)
| MICROSOFT_CLIENT_ID | Client ID for Microsoft services
| MICROSOFT_CLIENT_SECRET | Client secret for Microsoft services
| MICROSOFT_TENANT | Tenant ID for Microsoft Azure
| MICROSOFT_SERVICE_PRINCIPAL_ID | Service Principal ID for Microsoft Enterprise Application. (This is an advanced feature if you want litellm to auto-assign members to Litellm Teams based on their Microsoft Entra ID Groups)
| MICROSOFT_TENANT | Tenant ID for Microsoft Azure
| MICROSOFT_TOKEN_ENDPOINT | Custom token endpoint URL for Microsoft SSO (overrides default Microsoft OAuth token endpoint)
| MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE | Field name for user display name in Microsoft SSO response. Default is `displayName`
| MICROSOFT_USER_EMAIL_ATTRIBUTE | Field name for user email in Microsoft SSO response. Default is `userPrincipalName`
| MICROSOFT_USER_FIRST_NAME_ATTRIBUTE | Field name for user first name in Microsoft SSO response. Default is `givenName`
| MICROSOFT_USER_ID_ATTRIBUTE | Field name for user ID in Microsoft SSO response. Default is `id`
| MICROSOFT_USER_LAST_NAME_ATTRIBUTE | Field name for user last name in Microsoft SSO response. Default is `surname`
| MICROSOFT_USERINFO_ENDPOINT | Custom userinfo endpoint URL for Microsoft SSO (overrides default Microsoft Graph userinfo endpoint)
| NO_DOCS | Flag to disable Swagger UI documentation
| NO_REDOC | Flag to disable Redoc documentation
| NO_PROXY | List of addresses to bypass proxy
@ -718,6 +801,8 @@ router_settings:
| OPENMETER_API_ENDPOINT | API endpoint for OpenMeter integration
| OPENMETER_API_KEY | API key for OpenMeter services
| OPENMETER_EVENT_TYPE | Type of events sent to OpenMeter
| ONYX_API_BASE | Base URL for Onyx Security AI Guard service (defaults to https://ai-guard.onyx.security)
| ONYX_API_KEY | API key for Onyx Security AI Guard service
| OTEL_ENDPOINT | OpenTelemetry endpoint for traces
| OTEL_EXPORTER_OTLP_ENDPOINT | OpenTelemetry endpoint for traces
| OTEL_ENVIRONMENT_NAME | Environment name for OpenTelemetry
@ -728,6 +813,7 @@ router_settings:
| OTEL_EXPORTER_OTLP_HEADERS | Headers for OpenTelemetry requests
| OTEL_SERVICE_NAME | Service name identifier for OpenTelemetry
| OTEL_TRACER_NAME | Tracer name for OpenTelemetry tracing
| OTEL_LOGS_EXPORTER | Exporter type for OpenTelemetry logs (e.g., console)
| PAGERDUTY_API_KEY | API key for PagerDuty Alerting
| PANW_PRISMA_AIRS_API_KEY | API key for PANW Prisma AIRS service
| PANW_PRISMA_AIRS_API_BASE | Base URL for PANW Prisma AIRS service
@ -749,7 +835,7 @@ router_settings:
| PROMPTLAYER_API_KEY | API key for PromptLayer integration
| PROXY_ADMIN_ID | Admin identifier for proxy server
| PROXY_BASE_URL | Base URL for proxy service
| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 30
| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10
| PROXY_BATCH_POLLING_INTERVAL | Time in seconds to wait before polling a batch, to check if it's completed. Default is 6000s (1 hour)
| PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605
| PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597
@ -773,18 +859,18 @@ router_settings:
| REPLICATE_MODEL_NAME_WITH_ID_LENGTH | Length of Replicate model names with ID. Default is 64
| REPLICATE_POLLING_DELAY_SECONDS | Delay in seconds for Replicate polling operations. Default is 0.5
| REQUEST_TIMEOUT | Timeout in seconds for requests. Default is 6000
| ROOT_REDIRECT_URL | URL to redirect root path (/) to when DOCS_URL is set to something other than "/" (DOCS_URL is "/" by default)
| ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5
| RUNWAYML_DEFAULT_API_VERSION | Default API version for RunwayML service. Default is "2024-11-06"
| RUNWAYML_POLLING_TIMEOUT | Timeout in seconds for RunwayML image generation polling. Default is 600 (10 minutes)
| SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours)
| SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'.
| SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001.
| SUPERVISORD_STOPWAITSECS | Upper bound timeout in seconds for graceful shutdown when SEPARATE_HEALTH_APP=1. Default: 3600 (1 hour).
| SERVER_ROOT_PATH | Root path for the server application
| SEND_USER_API_KEY_ALIAS | Flag to send user API key alias to Zscaler AI Guard. Default is False
| SEND_USER_API_KEY_TEAM_ID | Flag to send user API key team ID to Zscaler AI Guard. Default is False
| SEND_USER_API_KEY_USER_ID | Flag to send user API key user ID to Zscaler AI Guard. Default is False
| SET_VERBOSE | Flag to enable verbose logging
| SET_VERBOSE | [DEPRECATED] Use `LITELLM_LOG` instead with values "INFO", "DEBUG", or "ERROR". See [debugging docs](./debugging)
| SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD | Minimum number of requests to consider "reasonable traffic" for single-deployment cooldown logic. Default is 1000
| SLACK_DAILY_REPORT_FREQUENCY | Frequency of daily Slack reports (e.g., daily, weekly)
| SLACK_WEBHOOK_URL | Webhook URL for Slack integration
@ -795,6 +881,9 @@ router_settings:
| SMTP_SENDER_LOGO | Logo used in emails sent via SMTP
| SMTP_TLS | Flag to enable or disable TLS for SMTP connections
| SMTP_USERNAME | Username for SMTP authentication (do not set if SMTP does not require auth)
| SENDGRID_API_KEY | API key for SendGrid email service
| RESEND_API_KEY | API key for Resend email service
| SENDGRID_SENDER_EMAIL | Email address used as the sender in SendGrid email transactions
| SPEND_LOGS_URL | URL for retrieving spend logs
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000
| SSL_CERTIFICATE | Path to the SSL certificate file
@ -826,12 +915,17 @@ router_settings:
| UPSTREAM_LANGFUSE_SECRET_KEY | Secret key for upstream Langfuse authentication
| USE_AWS_KMS | Flag to enable AWS Key Management Service for encryption
| USE_PRISMA_MIGRATE | Flag to use prisma migrate instead of prisma db push. Recommended for production environments.
| WANDB_API_KEY | API key for Weights & Biases (W&B) logging integration
| WANDB_HOST | Host URL for Weights & Biases (W&B) service
| WANDB_PROJECT_ID | Project ID for Weights & Biases (W&B) logging integration
| WEBHOOK_URL | URL for receiving webhooks from external services
| SPEND_LOG_RUN_LOOPS | Constant for setting how many runs of 1000 batch deletes should spend_log_cleanup task run
| SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000
| SPEND_LOG_QUEUE_POLL_INTERVAL | Polling interval in seconds for spend log queue. Default is 2.0
| SPEND_LOG_QUEUE_SIZE_THRESHOLD | Threshold for spend log queue size before processing. Default is 100
| COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY | Maximum size for CoroutineChecker in-memory cache. Default is 1000
| DEFAULT_SHARED_HEALTH_CHECK_TTL | Time-to-live in seconds for cached health check results in shared health check mode. Default is 300 (5 minutes)
| DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL | Time-to-live in seconds for health check lock in shared health check mode. Default is 60 (1 minute)
| ZSCALER_AI_GUARD_API_KEY | API key for Zscaler AI Guard service
| ZSCALER_AI_GUARD_POLICY_ID | Policy ID for Zscaler AI Guard guardrails
| ZSCALER_AI_GUARD_URL | Base URL for Zscaler AI Guard API. Default is https://api.us1.zseclipse.net/v1/detection/execute-policy
| ZSCALER_AI_GUARD_URL | Base URL for Zscaler AI Guard API. Default is https://api.us1.zseclipse.net/v1/detection/execute-policy

125
litellm/constants.py
View file

@ -1,4 +1,5 @@
import os
import sys
from typing import List, Literal
DEFAULT_HEALTH_CHECK_PROMPT = str(
@ -47,12 +48,20 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int(
DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
DEFAULT_IMAGE_WIDTH = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300))
DEFAULT_IMAGE_HEIGHT = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300))
# Maximum size for image URL downloads in MB (default 50MB, set to 0 to disable limit)
# This prevents memory issues from downloading very large images
# Maps to OpenAI's 50 MB payload limit - requests with images exceeding this size will be rejected
# Set MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0 to disable image URL handling entirely
MAX_IMAGE_URL_DOWNLOAD_SIZE_MB = float(os.getenv("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 50))
MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024)
) # 1MB = 1024KB
SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD = int(
os.getenv("SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD", 1000)
) # Minimum number of requests to consider "reasonable traffic". Used for single-deployment cooldown logic.
DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS = int(
os.getenv("DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS", 5)
) # Minimum number of requests before applying error rate cooldown. Prevents cooldown from triggering on first failure.
DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET = int(
os.getenv("DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET", 0)
@ -99,10 +108,18 @@ RUNWAYML_POLLING_TIMEOUT = int(
########## Networking constants ##############################################################
_DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour
# Aiohttp connection pooling constants
AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 0))
# Aiohttp connection pooling - prevents memory leaks from unbounded connection growth
# Set to 0 for unlimited (not recommended for production)
AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 300))
AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 50))
AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120))
AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300))
# enable_cleanup_closed is only needed for Python versions with the SSL leak bug
# Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960)
# Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78
AIOHTTP_NEEDS_CLEANUP_CLOSED = (
(3, 13, 0) <= sys.version_info < (3, 13, 1) or sys.version_info < (3, 12, 7)
)
# WebSocket constants
# Default to None (unlimited) to match OpenAI's official agents SDK behavior
@ -139,9 +156,12 @@ DEFAULT_SSL_CIPHERS = os.getenv(
REDIS_UPDATE_BUFFER_KEY = "litellm_spend_update_buffer"
REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_spend_update_buffer"
REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_team_spend_update_buffer"
REDIS_DAILY_ORG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_org_spend_update_buffer"
REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_end_user_spend_update_buffer"
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_agent_spend_update_buffer"
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer"
MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100))
MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 10000))
MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 2000))
MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int(
os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000)
)
@ -206,6 +226,7 @@ REPEATED_STREAMING_CHUNK_LIMIT = int(
os.getenv("REPEATED_STREAMING_CHUNK_LIMIT", 100)
) # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives.
DEFAULT_MAX_LRU_CACHE_SIZE = int(os.getenv("DEFAULT_MAX_LRU_CACHE_SIZE", 16))
_REALTIME_BODY_CACHE_SIZE = 1000 # Keep realtime helper caches bounded; workloads rarely exceed 1k models/intents
INITIAL_RETRY_DELAY = float(os.getenv("INITIAL_RETRY_DELAY", 0.5))
MAX_RETRY_DELAY = float(os.getenv("MAX_RETRY_DELAY", 8.0))
JITTER = float(os.getenv("JITTER", 0.75))
@ -253,12 +274,16 @@ TOGETHER_AI_EMBEDDING_350_M = int(os.getenv("TOGETHER_AI_EMBEDDING_350_M", 350))
QDRANT_SCALAR_QUANTILE = float(os.getenv("QDRANT_SCALAR_QUANTILE", 0.99))
QDRANT_VECTOR_SIZE = int(os.getenv("QDRANT_VECTOR_SIZE", 1536))
CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0.02))
AUDIO_SPEECH_CHUNK_SIZE = int(
os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192)
) # chunk_size for audio speech streaming. Balance between latency and memory usage
MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512)
)
DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
#### Networking settings ####
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds
DEFAULT_A2A_AGENT_TIMEOUT: float = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)) # 10 minutes
STREAM_SSE_DONE_STRING: str = "[DONE]"
STREAM_SSE_DATA_PREFIX: str = "data: "
### SPEND TRACKING ###
@ -275,12 +300,30 @@ REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM"
MAX_LANGFUSE_INITIALIZED_CLIENTS = int(
os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50)
)
LOGGING_WORKER_CONCURRENCY = int(
os.getenv("LOGGING_WORKER_CONCURRENCY", 100)
) # Must be above 0
LOGGING_WORKER_MAX_QUEUE_SIZE = int(os.getenv("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000))
LOGGING_WORKER_MAX_TIME_PER_COROUTINE = float(
os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0)
)
LOGGING_WORKER_CLEAR_PERCENTAGE = int(
os.getenv("LOGGING_WORKER_CLEAR_PERCENTAGE", 50)
) # Percentage of queue to clear (default: 50%)
MAX_ITERATIONS_TO_CLEAR_QUEUE = int(os.getenv("MAX_ITERATIONS_TO_CLEAR_QUEUE", 200))
MAX_TIME_TO_CLEAR_QUEUE = float(os.getenv("MAX_TIME_TO_CLEAR_QUEUE", 5.0))
LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS = float(
os.getenv("LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS", 0.5)
) # Cooldown time in seconds before allowing another aggressive clear (default: 0.5s)
DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE = os.getenv(
"DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE", "streaming.chunk.yield"
)
EMAIL_BUDGET_ALERT_TTL = int(os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60)) # 24 hours in seconds
EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE = float(os.getenv("EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8)) # 80% of max budget
############### LLM Provider Constants ###############
### ANTHROPIC CONSTANTS ###
ANTHROPIC_SKILLS_API_BETA_VERSION = "skills-2025-10-02"
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES = {
"low": 1,
"medium": 5,
@ -314,6 +357,7 @@ LITELLM_CHAT_PROVIDERS = [
"huggingface",
"together_ai",
"datarobot",
"helicone",
"openrouter",
"cometapi",
"vertex_ai",
@ -337,6 +381,7 @@ LITELLM_CHAT_PROVIDERS = [
"perplexity",
"mistral",
"groq",
"gigachat",
"nvidia_nim",
"cerebras",
"baseten",
@ -372,6 +417,7 @@ LITELLM_CHAT_PROVIDERS = [
"nebius",
"dashscope",
"moonshot",
"publicai",
"v0",
"heroku",
"oci",
@ -381,6 +427,8 @@ LITELLM_CHAT_PROVIDERS = [
"wandb",
"ovhcloud",
"lemonade",
"docker_model_runner",
"amazon_nova",
]
LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [
@ -506,6 +554,7 @@ openai_compatible_endpoints: List = [
"https://api.friendli.ai/serverless/v1",
"api.sambanova.ai/v1",
"api.x.ai/v1",
"ollama.com",
"api.galadriel.ai/v1",
"api.llama.com/compat/v1/",
"api.featherless.ai/v1",
@ -513,10 +562,17 @@ openai_compatible_endpoints: List = [
"api.studio.nebius.ai/v1",
"https://dashscope-intl.aliyuncs.com/compatible-mode/v1",
"https://api.moonshot.ai/v1",
"https://api.publicai.co/v1",
"https://api.synthetic.new/openai/v1",
"https://api.stima.tech/v1",
"https://nano-gpt.com/api/v1",
"https://api.poe.com/v1",
"https://llm.chutes.ai/v1/",
"https://api.v0.dev/v1",
"https://api.morphllm.com/v1",
"https://api.lambda.ai/v1",
"https://api.hyperbolic.xyz/v1",
"https://ai-gateway.helicone.ai/",
"https://ai-gateway.vercel.sh/v1",
"https://api.inference.wandb.ai/v1",
"https://api.clarifai.com/v2/ext/openai/v1",
@ -539,6 +595,7 @@ openai_compatible_providers: List = [
"perplexity",
"xinference",
"xai",
"zai",
"together_ai",
"fireworks_ai",
"empower",
@ -553,12 +610,19 @@ openai_compatible_providers: List = [
"github_copilot", # GitHub Copilot Chat API
"novita",
"meta_llama",
"publicai", # PublicAI - JSON-configured provider
"synthetic", # Synthetic - JSON-configured provider
"apertis", # Apertis - JSON-configured provider
"nano-gpt", # Nano-GPT - JSON-configured provider
"poe", # Poe - JSON-configured provider
"chutes", # Chutes - JSON-configured provider
"featherless_ai",
"nscale",
"nebius",
"dashscope",
"moonshot",
"v0",
"helicone",
"morph",
"lambda_ai",
"hyperbolic",
@ -567,6 +631,8 @@ openai_compatible_providers: List = [
"wandb",
"cometapi",
"clarifai",
"docker_model_runner",
"ragflow",
]
openai_text_completion_compatible_providers: List = (
[ # providers that support `/v1/completions`
@ -579,6 +645,12 @@ openai_text_completion_compatible_providers: List = (
"nebius",
"dashscope",
"moonshot",
"publicai",
"synthetic",
"apertis",
"nano-gpt",
"poe",
"chutes",
"v0",
"lambda_ai",
"hyperbolic",
@ -838,12 +910,18 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[
"nova",
"deepseek_r1",
"qwen3",
"qwen2",
"twelvelabs",
"openai",
"stability",
"moonshot",
]
BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[
"cohere",
"amazon",
"twelvelabs",
"nova",
]
BEDROCK_CONVERSE_MODELS = [
@ -886,6 +964,11 @@ BEDROCK_CONVERSE_MODELS = [
"meta.llama3-2-3b-instruct-v1:0",
"meta.llama3-2-11b-instruct-v1:0",
"meta.llama3-2-90b-instruct-v1:0",
"amazon.nova-lite-v1:0",
"amazon.nova-2-lite-v1:0",
"amazon.nova-pro-v1:0",
"writer.palmyra-x4-v1:0",
"writer.palmyra-x5-v1:0",
]
@ -904,6 +987,7 @@ cohere_embedding_models: set = set(
bedrock_embedding_models: set = set(
[
"amazon.titan-embed-text-v1",
"amazon.nova-2-multimodal-embeddings-v1:0",
"cohere.embed-english-v3",
"cohere.embed-multilingual-v3",
"cohere.embed-v4:0",
@ -994,6 +1078,13 @@ LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated"
########################### LiteLLM Proxy Specific Constants ###########################
########################################################################################
# Standard headers that are always checked for customer/end-user ID (no configuration required)
# These headers work out-of-the-box for tools like Claude Code that support custom headers
STANDARD_CUSTOMER_ID_HEADERS = [
"x-litellm-customer-id",
"x-litellm-end-user-id",
]
MAX_SPENDLOG_ROWS_TO_QUERY = int(
os.getenv("MAX_SPENDLOG_ROWS_TO_QUERY", 1_000_000)
) # if spendLogs has more than 1M rows, do not query the DB
@ -1044,6 +1135,8 @@ LITELLM_PROXY_ADMIN_NAME = "default_user_id"
########################### CLI SSO AUTHENTICATION CONSTANTS ###########################
LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli"
LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session"
CLI_JWT_TOKEN_NAME = "cli-jwt-token"
########################### DB CRON JOB NAMES ###########################
DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job"
@ -1055,6 +1148,8 @@ CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
SPEND_LOG_CLEANUP_JOB_NAME = "spend_log_cleanup"
SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
SPEND_LOG_QUEUE_POLL_INTERVAL = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS = int(
os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)
) # 1 minute
@ -1116,9 +1211,12 @@ SECRET_MANAGER_REFRESH_INTERVAL = int(
)
LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
"default_internal_user_params",
"public_mcp_servers",
"public_agent_groups",
"public_model_groups",
"public_model_groups_links",
"cost_discount_config",
"cost_margin_config",
]
SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"]
DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(
@ -1195,3 +1293,24 @@ SENTRY_PII_DENYLIST = [
COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY = int(
os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000)
)
########################### RAG Text Splitter Constants ###########################
DEFAULT_CHUNK_SIZE = int(os.getenv("DEFAULT_CHUNK_SIZE", 1000))
DEFAULT_CHUNK_OVERLAP = int(os.getenv("DEFAULT_CHUNK_OVERLAP", 200))
########################### Microsoft SSO Constants ###########################
MICROSOFT_USER_EMAIL_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_EMAIL_ATTRIBUTE", "userPrincipalName")
)
MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_DISPLAY_NAME_ATTRIBUTE", "displayName")
)
MICROSOFT_USER_ID_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_ID_ATTRIBUTE", "id")
)
MICROSOFT_USER_FIRST_NAME_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_FIRST_NAME_ATTRIBUTE", "givenName")
)
MICROSOFT_USER_LAST_NAME_ATTRIBUTE = str(
os.getenv("MICROSOFT_USER_LAST_NAME_ATTRIBUTE", "surname")
)

648
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -1,3 +1,4 @@
import base64
import copy
import hashlib
import json
@ -5,7 +6,7 @@ import mimetypes
import re
import xml.etree.ElementTree as ET
from enum import Enum
from typing import Any, List, Optional, Tuple, cast, overload
from typing import Any, Dict, List, Optional, Tuple, Union, cast, overload
from jinja2.sandbox import ImmutableSandboxedEnvironment
@ -43,6 +44,7 @@ from .common_utils import (
convert_content_list_to_str,
infer_content_type_from_url_and_content,
is_non_content_values_set,
parse_tool_call_arguments,
)
from .image_handling import convert_url_to_base64
@ -57,6 +59,10 @@ def prompt_injection_detection_default_pt():
BAD_MESSAGE_ERROR_STR = "Invalid Message "
# Separator used to embed Gemini thought signatures in tool call IDs
# See: https://ai.google.dev/gemini-api/docs/thought-signatures
THOUGHT_SIGNATURE_SEPARATOR = "__thought__"
# used to interweave user messages, to ensure user/assistant alternating
DEFAULT_USER_CONTINUE_MESSAGE = {
"role": "user",
@ -897,11 +903,70 @@ def convert_to_anthropic_image_obj(
media_type=media_type,
data=base64_data,
)
except litellm.ImageFetchError:
raise
except Exception as e:
if "Error: Unable to fetch image from URL" in str(e):
raise e
raise Exception(
"""Image url not in expected format. Example Expected input - "image_url": "data:image/jpeg;base64,{base64_image}". Supported formats - ['image/jpeg', 'image/png', 'image/gif', 'image/webp']."""
f"""Image url not in expected format. Example Expected input - "image_url": "data:image/jpeg;base64,{{base64_image}}". Supported formats - ['image/jpeg', 'image/png', 'image/gif', 'image/webp']. Error: {str(e)}"""
)
def create_anthropic_image_param(
image_url_input: Union[str, dict],
format: Optional[str] = None,
is_bedrock_invoke: bool = False,
) -> AnthropicMessagesImageParam:
"""
Create an AnthropicMessagesImageParam from an image URL input.
Supports both URL references (for HTTP/HTTPS URLs) and base64 encoding.
"""
# Extract URL and format from input
if isinstance(image_url_input, str):
image_url = image_url_input
else:
image_url = image_url_input.get("url", "")
if format is None:
format = image_url_input.get("format")
# Check if the image URL is an HTTP/HTTPS URL
if image_url.startswith("http://") or image_url.startswith("https://"):
# For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
# as these providers don't support URL sources for images
if is_bedrock_invoke or image_url.startswith("http://"):
base64_url = convert_url_to_base64(url=image_url)
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=base64_url, format=format
)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
else:
# HTTPS URL - pass directly for regular Anthropic
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSourceUrl(
type="url",
url=image_url,
),
)
else:
# Convert to base64 for data URIs or other formats
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=image_url, format=format
)
return AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
@ -967,9 +1032,11 @@ def convert_to_anthropic_tool_invoke_xml(tool_calls: list) -> str:
tool_function = get_attribute_or_key(tool, "function")
tool_name = get_attribute_or_key(tool_function, "name")
tool_arguments = get_attribute_or_key(tool_function, "arguments")
parsed_args = parse_tool_call_arguments(
tool_arguments, tool_name=tool_name, context="Anthropic XML tool invoke"
)
parameters = "".join(
f"<{param}>{val}</{param}>\n"
for param, val in json.loads(tool_arguments).items()
f"<{param}>{val}</{param}>\n" for param, val in parsed_args.items()
)
invokes += (
"<invoke>\n"
@ -1007,15 +1074,41 @@ def anthropic_messages_pt_xml(messages: list):
if isinstance(messages[msg_i]["content"], list):
for m in messages[msg_i]["content"]:
if m.get("type", "") == "image_url":
format = m["image_url"].get("format")
user_content.append(
{
"type": "image",
"source": convert_to_anthropic_image_obj(
m["image_url"]["url"], format=format
),
}
format = (
m["image_url"].get("format")
if isinstance(m["image_url"], dict)
else None
)
image_param = create_anthropic_image_param(
m["image_url"], format=format
)
# Convert to dict format for XML version
source = image_param["source"]
if isinstance(source, dict) and source.get("type") == "url":
# Type narrowing for URL source
url_source = cast(AnthropicContentParamSourceUrl, source)
user_content.append(
{
"type": "image",
"source": {
"type": "url",
"url": url_source["url"],
},
}
)
else:
# Type narrowing for base64 source
base64_source = cast(AnthropicContentParamSource, source)
user_content.append(
{
"type": "image",
"source": {
"type": "base64",
"media_type": base64_source["media_type"],
"data": base64_source["data"],
},
}
)
elif m.get("type", "") == "text":
user_content.append({"type": "text", "text": m["text"]})
else:
@ -1161,8 +1254,94 @@ def _gemini_tool_call_invoke_helper(
return function_call
def _encode_tool_call_id_with_signature(
tool_call_id: str, thought_signature: Optional[str]
) -> str:
"""
Embed thought signature into tool call ID for OpenAI client compatibility.
Args:
tool_call_id: The tool call ID (e.g., "call_abc123...")
thought_signature: Base64-encoded signature from Gemini response
Returns:
Tool call ID with embedded signature if present, otherwise original ID
Format: call_<uuid>__thought__<base64_signature>
See: https://ai.google.dev/gemini-api/docs/thought-signatures
"""
if thought_signature:
return f"{tool_call_id}{THOUGHT_SIGNATURE_SEPARATOR}{thought_signature}"
return tool_call_id
def _get_thought_signature_from_tool(
tool: dict, model: Optional[str] = None
) -> Optional[str]:
"""Extract thought signature from tool call's provider_specific_fields.
If not provided try to extract thought signature from tool call id
Checks both tool.provider_specific_fields and tool.function.provider_specific_fields.
If no signature is found and model is gemini-3, returns a dummy signature.
"""
# First check tool's provider_specific_fields
provider_fields = tool.get("provider_specific_fields") or {}
if isinstance(provider_fields, dict):
signature = provider_fields.get("thought_signature")
if signature:
return signature
# Then check function's provider_specific_fields
function = tool.get("function")
if function:
if isinstance(function, dict):
func_provider_fields = function.get("provider_specific_fields") or {}
if isinstance(func_provider_fields, dict):
signature = func_provider_fields.get("thought_signature")
if signature:
return signature
elif (
hasattr(function, "provider_specific_fields")
and function.provider_specific_fields
):
if isinstance(function.provider_specific_fields, dict):
signature = function.provider_specific_fields.get("thought_signature")
if signature:
return signature
# Check if thought signature is embedded in tool call ID
tool_call_id = tool.get("id")
if tool_call_id and THOUGHT_SIGNATURE_SEPARATOR in tool_call_id:
parts = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
if len(parts) == 2:
_, signature = parts
return signature
# If no signature found and model is gemini-3, return dummy signature
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
if model and VertexGeminiConfig._is_gemini_3_or_newer(model):
return _get_dummy_thought_signature()
return None
def _get_dummy_thought_signature() -> str:
"""Generate a dummy thought signature for models that require it.
This is used when transferring conversation history from older models
(like gemini-2.5-flash) to gemini-3, which requires thought_signature
for strict validation.
"""
# Return a base64-encoded dummy signature string
# Below dummy signature is recommended by google - https://ai.google.dev/gemini-api/docs/thought-signatures#faqs
dummy_data = b"skip_thought_signature_validator"
return base64.b64encode(dummy_data).decode("utf-8")
def convert_to_gemini_tool_call_invoke(
message: ChatCompletionAssistantMessage,
model: Optional[str] = None,
) -> List[VertexPartType]:
"""
OpenAI tool invokes:
@ -1207,8 +1386,9 @@ def convert_to_gemini_tool_call_invoke(
_parts_list: List[VertexPartType] = []
tool_calls = message.get("tool_calls", None)
function_call = message.get("function_call", None)
if tool_calls is not None:
for tool in tool_calls:
for idx, tool in enumerate(tool_calls):
if "function" in tool:
gemini_function_call: Optional[VertexFunctionCall] = (
_gemini_tool_call_invoke_helper(
@ -1216,9 +1396,16 @@ def convert_to_gemini_tool_call_invoke(
)
)
if gemini_function_call is not None:
_parts_list.append(
VertexPartType(function_call=gemini_function_call)
part_dict: VertexPartType = {
"function_call": gemini_function_call
}
thought_signature = _get_thought_signature_from_tool(
dict(tool), model=model
)
if thought_signature:
part_dict["thoughtSignature"] = thought_signature
_parts_list.append(part_dict)
else: # don't silently drop params. Make it clear to user what's happening.
raise Exception(
"function_call missing. Received tool call with 'type': 'function'. No function call in argument - {}".format(
@ -1230,7 +1417,36 @@ def convert_to_gemini_tool_call_invoke(
function_call_params=function_call
)
if gemini_function_call is not None:
_parts_list.append(VertexPartType(function_call=gemini_function_call))
part_dict_function: VertexPartType = {
"function_call": gemini_function_call
}
# Extract thought signature from function_call's provider_specific_fields
thought_signature = None
provider_fields = (
function_call.get("provider_specific_fields")
if isinstance(function_call, dict)
else {}
)
if isinstance(provider_fields, dict):
thought_signature = provider_fields.get("thought_signature")
# If no signature found and model is gemini-3, use dummy signature
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
if (
not thought_signature
and model
and VertexGeminiConfig._is_gemini_3_or_newer(model)
):
thought_signature = _get_dummy_thought_signature()
if thought_signature:
part_dict_function["thoughtSignature"] = thought_signature
_parts_list.append(part_dict_function)
else: # don't silently drop params. Make it clear to user what's happening.
raise Exception(
"function_call missing. Received tool call with 'type': 'function'. No function call in argument - {}".format(
@ -1249,7 +1465,7 @@ def convert_to_gemini_tool_call_invoke(
def convert_to_gemini_tool_call_result(
message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
last_message_with_tool_calls: Optional[dict],
) -> VertexPartType:
) -> Union[VertexPartType, List[VertexPartType]]:
"""
OpenAI message with a tool result looks like:
{
@ -1265,16 +1481,54 @@ def convert_to_gemini_tool_call_result(
"name": "get_current_weather",
"content": "function result goes here",
}
Supports content with images for Computer Use:
{
"role": "tool",
"tool_call_id": "call_abc123",
"content": [
{"type": "text", "text": "I found the requested image:"},
{"type": "input_image", "image_url": "https://example.com/image.jpg" }
]
}
"""
from litellm.types.llms.vertex_ai import BlobType
content_str: str = ""
inline_data: Optional[BlobType] = None
if "content" in message:
if isinstance(message["content"], str):
content_str = message["content"]
elif isinstance(message["content"], List):
content_list = message["content"]
for content in content_list:
if content["type"] == "text":
content_str += content["text"]
content_type = content.get("type", "")
if content_type == "text":
content_str += content.get("text", "")
elif content_type in ("input_image", "image_url"):
# Extract image for inline_data (for Computer Use screenshots and tool results)
image_url_data = content.get("image_url", "")
image_url = (
image_url_data.get("url", "")
if isinstance(image_url_data, dict)
else image_url_data
)
if image_url:
# Convert image to base64 blob format for Gemini
try:
image_obj = convert_to_anthropic_image_obj(
image_url, format=None
)
inline_data = BlobType(
data=image_obj["data"],
mime_type=image_obj["media_type"],
)
except Exception as e:
verbose_logger.warning(
f"Failed to process image in tool response: {e}"
)
name: Optional[str] = message.get("name", "") # type: ignore
# Recover name from last message with tool calls
@ -1297,17 +1551,58 @@ def convert_to_gemini_tool_call_result(
)
)
# Parse response data - support both JSON string and plain string
# For Computer Use, the response should contain structured data like {"url": "..."}
response_data: dict
try:
if content_str.strip().startswith("{") or content_str.strip().startswith("["):
# Try to parse as JSON (for Computer Use structured responses)
parsed = json.loads(content_str)
if isinstance(parsed, dict):
response_data = parsed # Use the parsed JSON directly
else:
response_data = {"content": content_str}
else:
response_data = {"content": content_str}
except (json.JSONDecodeError, ValueError):
# Not valid JSON, wrap in content field
response_data = {"content": content_str}
# We can't determine from openai message format whether it's a successful or
# error call result so default to the successful result template
_function_response = VertexFunctionResponse(
name=name, response={"content": content_str} # type: ignore
name=name, response=response_data # type: ignore
)
_part = VertexPartType(function_response=_function_response)
# Create part with function_response, and optionally inline_data for images (Computer Use)
_part: VertexPartType = {"function_response": _function_response}
# For Computer Use, if we have an image, we need separate parts:
# - One part with function_response
# - One part with inline_data
# Gemini's PartType is a oneof, so we can't have both in the same part
if inline_data:
image_part: VertexPartType = {"inline_data": inline_data}
return [_part, image_part]
return _part
def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
"""
Sanitize tool_use_id to match Anthropic's required pattern: ^[a-zA-Z0-9_-]+$
Anthropic requires tool_use_id to only contain alphanumeric characters, underscores, and hyphens.
This function replaces any invalid characters with underscores.
"""
# Replace any character that's not alphanumeric, underscore, or hyphen with underscore
sanitized = re.sub(r"[^a-zA-Z0-9_-]", "_", tool_use_id)
# Ensure it's not empty (fallback to a default if needed)
if not sanitized:
sanitized = "tool_use_id"
return sanitized
def convert_to_anthropic_tool_result(
message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
) -> AnthropicMessagesToolResultParam:
@ -1363,25 +1658,19 @@ def convert_to_anthropic_tool_result(
)
)
elif content["type"] == "image_url":
if isinstance(content["image_url"], str):
image_chunk = convert_to_anthropic_image_obj(
content["image_url"], format=None
)
else:
format = content["image_url"].get("format")
image_chunk = convert_to_anthropic_image_obj(
content["image_url"]["url"], format=format
)
anthropic_content_list.append(
AnthropicMessagesImageParam(
type="image",
source=AnthropicContentParamSource(
type="base64",
media_type=image_chunk["media_type"],
data=image_chunk["data"],
),
)
format = (
content["image_url"].get("format")
if isinstance(content["image_url"], dict)
else None
)
_anthropic_image_param = create_anthropic_image_param(
content["image_url"], format=format
)
_anthropic_image_param = add_cache_control_to_content(
anthropic_content_element=_anthropic_image_param,
original_content_element=content,
)
anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
anthropic_content = anthropic_content_list
anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None
@ -1390,18 +1679,26 @@ def convert_to_anthropic_tool_result(
if message["role"] == "tool":
tool_message: ChatCompletionToolMessage = message
tool_call_id: str = tool_message["tool_call_id"]
# Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$
sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id)
# We can't determine from openai message format whether it's a successful or
# error call result so default to the successful result template
anthropic_tool_result = AnthropicMessagesToolResultParam(
type="tool_result", tool_use_id=tool_call_id, content=anthropic_content
type="tool_result",
tool_use_id=sanitized_tool_use_id,
content=anthropic_content,
)
if message["role"] == "function":
function_message: ChatCompletionFunctionMessage = message
tool_call_id = function_message.get("tool_call_id") or str(uuid.uuid4())
# Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$
sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id)
anthropic_tool_result = AnthropicMessagesToolResultParam(
type="tool_result", tool_use_id=tool_call_id, content=anthropic_content
type="tool_result",
tool_use_id=sanitized_tool_use_id,
content=anthropic_content,
)
if anthropic_tool_result is None:
@ -1417,12 +1714,17 @@ def convert_function_to_anthropic_tool_invoke(
try:
_name = get_attribute_or_key(function_call, "name") or ""
_arguments = get_attribute_or_key(function_call, "arguments")
tool_input = parse_tool_call_arguments(
_arguments, tool_name=_name, context="Anthropic function to tool invoke"
)
anthropic_tool_invoke = [
AnthropicMessagesToolUseParam(
type="tool_use",
id=str(uuid.uuid4()),
name=_name,
input=json.loads(_arguments) if _arguments else {},
input=tool_input,
)
]
return anthropic_tool_invoke
@ -1432,7 +1734,8 @@ def convert_function_to_anthropic_tool_invoke(
def convert_to_anthropic_tool_invoke(
tool_calls: List[ChatCompletionAssistantToolCall],
) -> List[AnthropicMessagesToolUseParam]:
web_search_results: Optional[List[Any]] = None,
) -> List[Union[AnthropicMessagesToolUseParam, Dict[str, Any]]]:
"""
OpenAI tool invokes:
{
@ -1468,38 +1771,70 @@ def convert_to_anthropic_tool_invoke(
}
]
}
For server-side tools (web_search), we need to reconstruct:
- server_tool_use blocks (id starts with "srvtoolu_")
- web_search_tool_result blocks (from provider_specific_fields)
Fixes: https://github.com/BerriAI/litellm/issues/17737
"""
anthropic_tool_invoke = []
anthropic_tool_invoke: List[
Union[AnthropicMessagesToolUseParam, Dict[str, Any]]
] = []
for tool in tool_calls:
if not get_attribute_or_key(tool, "type") == "function":
continue
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
type="tool_use",
id=cast(str, get_attribute_or_key(tool, "id")),
name=cast(
str,
get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"),
),
input=json.loads(
get_attribute_or_key(
get_attribute_or_key(tool, "function"), "arguments"
)
),
tool_id = cast(str, get_attribute_or_key(tool, "id"))
tool_name = cast(
str,
get_attribute_or_key(get_attribute_or_key(tool, "function"), "name"),
)
tool_input = parse_tool_call_arguments(
get_attribute_or_key(get_attribute_or_key(tool, "function"), "arguments"),
tool_name=tool_name,
context="Anthropic tool invoke",
)
_content_element = add_cache_control_to_content(
anthropic_content_element=_anthropic_tool_use_param,
original_content_element=dict(tool),
)
# Check if this is a server-side tool (web_search, tool_search, etc.)
# Server tool IDs start with "srvtoolu_"
if tool_id.startswith("srvtoolu_"):
# Create server_tool_use block instead of tool_use
_anthropic_server_tool_use: Dict[str, Any] = {
"type": "server_tool_use",
"id": tool_id,
"name": tool_name,
"input": tool_input,
}
anthropic_tool_invoke.append(_anthropic_server_tool_use)
if "cache_control" in _content_element:
_anthropic_tool_use_param["cache_control"] = _content_element[
"cache_control"
]
# Add corresponding web_search_tool_result if available
if web_search_results:
for result in web_search_results:
if result.get("tool_use_id") == tool_id:
anthropic_tool_invoke.append(result)
break
else:
# Regular tool_use
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
type="tool_use",
id=tool_id,
name=tool_name,
input=tool_input,
)
anthropic_tool_invoke.append(_anthropic_tool_use_param)
_content_element = add_cache_control_to_content(
anthropic_content_element=_anthropic_tool_use_param,
original_content_element=dict(tool),
)
if "cache_control" in _content_element:
_anthropic_tool_use_param["cache_control"] = _content_element[
"cache_control"
]
anthropic_tool_invoke.append(_anthropic_tool_use_param)
return anthropic_tool_invoke
@ -1711,20 +2046,36 @@ def anthropic_messages_pt( # noqa: PLR0915
for m in user_message_types_block["content"]:
if m.get("type", "") == "image_url":
m = cast(ChatCompletionImageObject, m)
format: Optional[str] = None
if isinstance(m["image_url"], str):
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=m["image_url"], format=None
format = (
m["image_url"].get("format")
if isinstance(m["image_url"], dict)
else None
)
# Convert ChatCompletionImageUrlObject to dict if needed
image_url_value = m["image_url"]
if isinstance(image_url_value, str):
image_url_input: Union[str, dict[str, Any]] = (
image_url_value
)
else:
format = m["image_url"].get("format")
image_chunk = convert_to_anthropic_image_obj(
openai_image_url=m["image_url"]["url"],
format=format,
)
_anthropic_content_element = (
_anthropic_content_element_factory(image_chunk)
# ChatCompletionImageUrlObject or dict case - convert to dict
image_url_input = {
"url": image_url_value["url"],
"format": image_url_value.get("format"),
}
# Bedrock invoke models have format: invoke/...
# Vertex AI Anthropic also doesn't support URL sources for images
is_bedrock_invoke = model.lower().startswith("invoke/")
is_vertex_ai = (
llm_provider.startswith("vertex_ai")
if llm_provider
else False
)
force_base64 = is_bedrock_invoke or is_vertex_ai
_anthropic_content_element = create_anthropic_image_param(
image_url_input,
format=format,
is_bedrock_invoke=force_base64,
)
_content_element = add_cache_control_to_content(
anthropic_content_element=_anthropic_content_element,
@ -1832,6 +2183,14 @@ def anthropic_messages_pt( # noqa: PLR0915
assistant_content.append(
cast(AnthropicMessagesTextParam, _cached_message)
)
# handle server_tool_use blocks (tool search, web search, etc.)
# Pass through as-is since these are Anthropic-native content types
elif m.get("type", "") == "server_tool_use":
assistant_content.append(m) # type: ignore
# handle tool_search_tool_result blocks
# Pass through as-is since these are Anthropic-native content types
elif m.get("type", "") == "tool_search_tool_result":
assistant_content.append(m) # type: ignore
elif (
"content" in assistant_content_block
and isinstance(assistant_content_block["content"], str)
@ -1860,8 +2219,29 @@ def anthropic_messages_pt( # noqa: PLR0915
if (
assistant_tool_calls is not None
): # support assistant tool invoke conversion
# Get web_search_results from provider_specific_fields for server_tool_use reconstruction
# Fixes: https://github.com/BerriAI/litellm/issues/17737
_provider_specific_fields_raw = assistant_content_block.get(
"provider_specific_fields"
)
_provider_specific_fields: Dict[str, Any] = {}
if isinstance(_provider_specific_fields_raw, dict):
_provider_specific_fields = cast(
Dict[str, Any], _provider_specific_fields_raw
)
_web_search_results = _provider_specific_fields.get(
"web_search_results"
)
tool_invoke_results = convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_web_search_results,
)
# AnthropicMessagesAssistantMessageValues includes AnthropicMessagesToolUseParam
assistant_content.extend(
convert_to_anthropic_tool_invoke(assistant_tool_calls)
cast(
List[AnthropicMessagesAssistantMessageValues],
tool_invoke_results,
)
)
assistant_function_call = assistant_content_block.get("function_call")
@ -2496,7 +2876,6 @@ def stringify_json_tool_call_content(messages: List) -> List:
###### AMAZON BEDROCK #######
import base64
from email.message import Message
import httpx
@ -2541,17 +2920,19 @@ class BedrockImageProcessor:
"""Handles both sync and async image processing for Bedrock conversations."""
@staticmethod
def _post_call_image_processing(response: httpx.Response, image_url: str = "") -> Tuple[str, str]:
def _post_call_image_processing(
response: httpx.Response, image_url: str = ""
) -> Tuple[str, str]:
# Check the response's content type to ensure it is an image
content_type = response.headers.get("content-type")
# Use helper function to infer content type with fallback logic
content_type = infer_content_type_from_url_and_content(
url=image_url,
content=response.content,
current_content_type=content_type,
)
content_type = _parse_content_type(content_type)
# Convert the image content to base64 bytes
@ -2570,7 +2951,9 @@ class BedrockImageProcessor:
response = await client.get(image_url, follow_redirects=True)
response.raise_for_status() # Raise an exception for HTTP errors
return BedrockImageProcessor._post_call_image_processing(response, image_url)
return BedrockImageProcessor._post_call_image_processing(
response, image_url
)
except Exception as e:
raise e
@ -2583,7 +2966,9 @@ class BedrockImageProcessor:
response = client.get(image_url, follow_redirects=True)
response.raise_for_status() # Raise an exception for HTTP errors
return BedrockImageProcessor._post_call_image_processing(response, image_url)
return BedrockImageProcessor._post_call_image_processing(
response, image_url
)
except Exception as e:
raise e
@ -2846,6 +3231,11 @@ def _convert_to_bedrock_tool_call_invoke(
id = tool["id"]
name = tool["function"].get("name", "")
arguments = tool["function"].get("arguments", "")
arguments_dict = json.loads(arguments) if arguments else {}
# Ensure arguments_dict is always a dict (Bedrock requires toolUse.input to be an object)
# When some providers return arguments: '""' (JSON-encoded empty string), json.loads returns ""
if not isinstance(arguments_dict, dict):
arguments_dict = {}
if not arguments or not arguments.strip():
arguments_dict = {}
else:
@ -2914,21 +3304,39 @@ def _convert_to_bedrock_tool_call_result(
"""
-
"""
content_str: str = ""
tool_result_content_blocks: List[BedrockToolResultContentBlock] = []
if isinstance(message["content"], str):
content_str = message["content"]
tool_result_content_blocks.append(
BedrockToolResultContentBlock(text=message["content"])
)
elif isinstance(message["content"], List):
content_list = message["content"]
for content in content_list:
if content["type"] == "text":
content_str += content["text"]
tool_result_content_blocks.append(
BedrockToolResultContentBlock(text=content["text"])
)
elif content["type"] == "image_url":
format: Optional[str] = None
if isinstance(content["image_url"], dict):
image_url = content["image_url"]["url"]
format = content["image_url"].get("format")
else:
image_url = content["image_url"]
_block: BedrockContentBlock = BedrockImageProcessor.process_image_sync(
image_url=image_url,
format=format,
)
if "image" in _block:
tool_result_content_blocks.append(
BedrockToolResultContentBlock(image=_block["image"])
)
message.get("name", "")
id = str(message.get("tool_call_id", str(uuid.uuid4())))
tool_result_content_block = BedrockToolResultContentBlock(text=content_str)
tool_result = BedrockToolResultBlock(
content=[tool_result_content_block],
content=tool_result_content_blocks,
toolUseId=id,
)
@ -3237,8 +3645,25 @@ class BedrockConverseMessagesProcessor:
@staticmethod
def _initial_message_setup(
messages: List,
model: str,
llm_provider: str,
user_continue_message: Optional[ChatCompletionUserMessage] = None,
) -> List:
# gracefully handle base case of no messages at all
if len(messages) == 0:
if user_continue_message is not None:
messages.append(user_continue_message)
elif litellm.modify_params:
messages.append(DEFAULT_USER_CONTINUE_MESSAGE)
else:
raise litellm.BadRequestError(
message=BAD_MESSAGE_ERROR_STR
+ "bedrock requires at least one non-system message",
model=model,
llm_provider=llm_provider,
)
# if initial message is assistant message
if messages[0].get("role") is not None and messages[0]["role"] == "assistant":
if user_continue_message is not None:
messages.insert(0, user_continue_message)
@ -3266,18 +3691,8 @@ class BedrockConverseMessagesProcessor:
contents: List[BedrockMessageBlock] = []
msg_i = 0
## BASE CASE ##
if len(messages) == 0:
raise litellm.BadRequestError(
message=BAD_MESSAGE_ERROR_STR
+ "bedrock requires at least one non-system message",
model=model,
llm_provider=llm_provider,
)
# if initial message is assistant message
messages = BedrockConverseMessagesProcessor._initial_message_setup(
messages, user_continue_message
messages, model, llm_provider, user_continue_message
)
while msg_i < len(messages):
@ -3638,28 +4053,9 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
contents: List[BedrockMessageBlock] = []
msg_i = 0
## BASE CASE ##
if len(messages) == 0:
raise litellm.BadRequestError(
message=BAD_MESSAGE_ERROR_STR
+ "bedrock requires at least one non-system message",
model=model,
llm_provider=llm_provider,
)
# if initial message is assistant message
if messages[0].get("role") is not None and messages[0]["role"] == "assistant":
if user_continue_message is not None:
messages.insert(0, user_continue_message)
elif litellm.modify_params:
messages.insert(0, DEFAULT_USER_CONTINUE_MESSAGE)
# if final message is assistant message
if messages[-1].get("role") is not None and messages[-1]["role"] == "assistant":
if user_continue_message is not None:
messages.append(user_continue_message)
elif litellm.modify_params:
messages.append(DEFAULT_USER_CONTINUE_MESSAGE)
messages = BedrockConverseMessagesProcessor._initial_message_setup(
messages, model, llm_provider, user_continue_message
)
while msg_i < len(messages):
user_content: List[BedrockContentBlock] = []
@ -3840,7 +4236,9 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
)
elif element["type"] == "text":
# AWS Bedrock doesn't allow empty or whitespace-only text content, so use placeholder for empty strings
text_content = element["text"] if element["text"].strip() else "."
text_content = (
element["text"] if element["text"].strip() else "."
)
assistants_part = BedrockContentBlock(text=text_content)
assistants_parts.append(assistants_part)
elif element["type"] == "image_url":

31
litellm/litellm_core_utils/prompt_templates/image_handling.py
View file

@ -9,6 +9,7 @@ from httpx import Response
import litellm
from litellm import verbose_logger
from litellm.caching.caching import InMemoryCache
from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB
MAX_IMGS_IN_MEMORY = 10
@ -21,7 +22,25 @@ def _process_image_response(response: Response, url: str) -> str:
f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}"
)
# Check size before downloading if Content-Length header is present
content_length = response.headers.get("Content-Length")
if content_length is not None:
size_mb = int(content_length) / (1024 * 1024)
if size_mb > MAX_IMAGE_URL_DOWNLOAD_SIZE_MB:
raise litellm.ImageFetchError(
f"Error: Image size ({size_mb:.2f}MB) exceeds maximum allowed size ({MAX_IMAGE_URL_DOWNLOAD_SIZE_MB}MB). url={url}"
)
image_bytes = response.content
# Check actual size after download if Content-Length was not available
if content_length is None:
size_mb = len(image_bytes) / (1024 * 1024)
if size_mb > MAX_IMAGE_URL_DOWNLOAD_SIZE_MB:
raise litellm.ImageFetchError(
f"Error: Image size ({size_mb:.2f}MB) exceeds maximum allowed size ({MAX_IMAGE_URL_DOWNLOAD_SIZE_MB}MB). url={url}"
)
base64_image = base64.b64encode(image_bytes).decode("utf-8")
image_type = response.headers.get("Content-Type")
@ -48,6 +67,12 @@ def _process_image_response(response: Response, url: str) -> str:
async def async_convert_url_to_base64(url: str) -> str:
# If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
raise litellm.ImageFetchError(
f"Error: Image URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}"
)
cached_result = in_memory_cache.get_cache(url)
if cached_result:
return cached_result
@ -67,6 +92,12 @@ async def async_convert_url_to_base64(url: str) -> str:
def convert_url_to_base64(url: str) -> str:
# If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
raise litellm.ImageFetchError(
f"Error: Image URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}"
)
cached_result = in_memory_cache.get_cache(url)
if cached_result:
return cached_result

101
litellm/proxy/proxy_config.yaml
View file

@ -1,67 +1,52 @@
model_list:
- model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1
- model_name: gemini/*
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-east-1
custom_llm_provider: bedrock
- model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1
model: gemini/*
- model_name: claude-sonnet-4-5-20250929
litellm_params:
model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0
aws_region_name: us-west-2
custom_llm_provider: bedrock
- model_name: bedrock/*
litellm_params:
model: bedrock/*
custom_llm_provider: bedrock
aws_region_name: us-west-2
- model_name: runwayml/*
litellm_params:
model: runwayml/*
model: bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0
model_info:
cache_creation_input_token_cost: 3.75e-06
cache_read_input_token_cost: 3e-07
input_cost_per_token: 3e-06
input_cost_per_token_above_200k_tokens: 6e-06
output_cost_per_token_above_200k_tokens: 2.25e-05
cache_creation_input_token_cost_above_200k_tokens: 7.5e-06
cache_read_input_token_cost_above_200k_tokens: 6e-07
litellm_provider: bedrock_converse
max_input_tokens: 200000
max_output_tokens: 64000
max_tokens: 200000
mode: chat
output_cost_per_token: 1.5e-05
search_context_cost_per_query:
search_context_size_high: 0.01
search_context_size_low: 0.01
search_context_size_medium: 0.01
supports_assistant_prefill: true
supports_computer_use: true
supports_function_calling: true
supports_pdf_input: true
supports_prompt_caching: true
supports_reasoning: true
supports_response_schema: true
supports_tool_choice: true
supports_vision: true
tool_use_system_prompt_tokens: 346
# like MCPs/vector stores
search_tools:
- search_tool_name: litellm-search
- model_name: us.anthropic.claude-sonnet-4-20250514-v1:0
litellm_params:
search_provider: perplexity
api_key: os.environ/PERPLEXITYAI_API_KEY
- search_tool_name: exa-search
model: bedrock/converse/us.anthropic.claude-sonnet-4-20250514-v1:0
model_info:
litellm_provider: bedrock_converse
mode: chat
- model_name: azure-claude-opus-4-5
litellm_params:
search_provider: exa_ai
api_key: os.environ/EXA_API_KEY
litellm_settings:
max_end_user_budget_id: "2f6634cd-c631-4d3b-96c7-ad510ea06eaf"
# Comprehensive logging settings
store_audit_logs: true
verbose: true
log_level: "DEBUG" # Options: DEBUG, INFO, WARNING, ERROR
callbacks: ["s3_v2", "smtp_email"]
s3_callback_params:
s3_endpoint_url: "https://localhost:443" # Replace with your Minio server URL and port
s3_aws_access_key_id: "minioadmin"
s3_aws_secret_access_key: "minioadmin"
s3_region_name: "minio" # This can be any value for Minio
s3_bucket_name: "litellm-test" # Replace with your bucket name
s3_use_ssl: False
s3_verify: False
cache: True
cache_params:
type: local
drop_params: True
model: azure_ai/claude-opus-4-5
api_base: https://krish-mh44t553-eastus2.services.ai.azure.com
api_key: os.environ/AZURE_ANTHROPIC_API_KEY
general_settings:
store_prompts_in_spend_logs: True
vector_store_registry:
- vector_store_name: "bedrock-litellm-website-knowledgebase"
litellm_params:
vector_store_id: "T37J8R4WTM"
custom_llm_provider: "bedrock"
vector_store_description: "Bedrock vector store for the Litellm website knowledgebase"
vector_store_metadata:
source: "https://www.litellm.com/docs"
store_prompts_in_spend_logs: true
forward_client_headers_to_llm_api: true

212
tests/llm_translation/test_gemini.py
View file

@ -295,6 +295,7 @@ def test_gemini_image_generation():
[
"gemini/gemini-2.5-flash-image-preview",
"gemini/gemini-2.0-flash-preview-image-generation",
"gemini/gemini-3-pro-image-preview",
],
)
def test_gemini_flash_image_preview_models(model_name: str):
@ -737,6 +738,11 @@ async def test_gemini_image_generation_async():
CONTENT = response.choices[0].message.content
# Check if images list exists and has items before accessing
assert hasattr(response.choices[0].message, "images"), "Response message should have images attribute"
assert response.choices[0].message.images is not None, "Images should not be None"
assert len(response.choices[0].message.images) > 0, "Images list should not be empty"
IMAGE_URL = response.choices[0].message.images[0]["image_url"]
print("IMAGE_URL: ", IMAGE_URL)
@ -1223,3 +1229,209 @@ def test_gemini_function_args_preserve_unicode():
assert parsed_args["recipient"] == "José"
assert "\\u" not in arguments_str
assert "José" in arguments_str
def test_anthropic_thinking_param_to_gemini_3_thinkingLevel():
"""
Test that Anthropic thinking parameters are correctly transformed to Gemini 3 thinkingLevel
instead of thinkingBudget.
For Gemini 3+ models (gemini-3-flash, gemini-3-pro, gemini-3-flash-preview):
- Should use thinkingLevel instead of thinkingBudget
- budget_tokens should map to thinkingLevel
Related issue: https://github.com/BerriAI/litellm/issues/XXXX
"""
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
from litellm.types.llms.anthropic import AnthropicThinkingParam
# Test 1: Anthropic thinking enabled with budget_tokens for Gemini 3 model
thinking_param: AnthropicThinkingParam = {
"type": "enabled",
"budget_tokens": 10000,
}
result = VertexGeminiConfig._map_thinking_param(
thinking_param=thinking_param,
model="gemini-3-flash",
)
# For Gemini 3, should use thinkingLevel, not thinkingBudget
assert "thinkingLevel" in result, "Should have thinkingLevel for Gemini 3"
assert "thinkingBudget" not in result, "Should NOT have thinkingBudget for Gemini 3"
assert result["includeThoughts"] is True
assert result["thinkingLevel"] in ["minimal", "low"], "thinkingLevel should be 'minimal' or 'low'"
# Test 2: Anthropic thinking disabled for Gemini 3
thinking_param_disabled: AnthropicThinkingParam = {
"type": "disabled",
"budget_tokens": None,
}
result_disabled = VertexGeminiConfig._map_thinking_param(
thinking_param=thinking_param_disabled,
model="gemini-3-pro-preview",
)
assert result_disabled.get("includeThoughts") is False
assert "thinkingLevel" not in result_disabled or result_disabled.get("thinkingLevel") is None
# Test 3: Budget tokens = 0 for Gemini 3
thinking_param_zero: AnthropicThinkingParam = {
"type": "enabled",
"budget_tokens": 0,
}
result_zero = VertexGeminiConfig._map_thinking_param(
thinking_param=thinking_param_zero,
model="gemini-3-flash",
)
assert result_zero["includeThoughts"] is False
assert "thinkingLevel" not in result_zero or result_zero.get("thinkingLevel") is None
# Test 4: Fiercefalcon model (Gemini 3 Flash checkpoint) should use thinkingLevel
result_gemini3flashpreview = VertexGeminiConfig._map_thinking_param(
thinking_param=thinking_param,
model="gemini-3-flash-preview",
)
assert "thinkingLevel" in result_gemini3flashpreview, "Should have thinkingLevel for gemini-3-flash-preview"
assert "thinkingBudget" not in result_gemini3flashpreview, "Should NOT have thinkingBudget for gemini-3-flash-preview"
assert result_gemini3flashpreview["includeThoughts"] is True
def test_anthropic_thinking_param_to_gemini_2_thinkingBudget():
"""
Test that Anthropic thinking parameters are correctly transformed to Gemini 2 thinkingBudget
(not thinkingLevel).
For Gemini 2.x models (gemini-2.5-flash, gemini-2.0-flash):
- Should continue using thinkingBudget
- thinkingLevel should NOT be used
Related issue: https://github.com/BerriAI/litellm/issues/XXXX
"""
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
from litellm.types.llms.anthropic import AnthropicThinkingParam
# Test 1: Anthropic thinking enabled with budget_tokens for Gemini 2 model
thinking_param: AnthropicThinkingParam = {
"type": "enabled",
"budget_tokens": 10000,
}
result = VertexGeminiConfig._map_thinking_param(
thinking_param=thinking_param,
model="gemini-2.5-flash",
)
# For Gemini 2, should use thinkingBudget, not thinkingLevel
assert "thinkingBudget" in result, "Should have thinkingBudget for Gemini 2"
assert "thinkingLevel" not in result, "Should NOT have thinkingLevel for Gemini 2"
assert result["includeThoughts"] is True
assert result["thinkingBudget"] == 10000
# Test 2: Anthropic thinking enabled for gemini-2.0-flash model
result_gemini2 = VertexGeminiConfig._map_thinking_param(
thinking_param=thinking_param,
model="gemini-2.0-flash-thinking-exp-01-21",
)
assert "thinkingBudget" in result_gemini2, "Should have thinkingBudget for Gemini 2"
assert "thinkingLevel" not in result_gemini2, "Should NOT have thinkingLevel for Gemini 2"
assert result_gemini2["includeThoughts"] is True
assert result_gemini2["thinkingBudget"] == 10000
def test_anthropic_thinking_param_via_map_openai_params():
"""
Test that the thinking parameter is correctly transformed through the full map_openai_params flow
for Gemini 3 models, resulting in thinkingConfig with thinkingLevel.
This tests the full integration from Anthropic API format to Gemini format.
"""
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig,
)
from litellm.types.llms.anthropic import AnthropicThinkingParam
config = VertexGeminiConfig()
# Test with Gemini 3 model
non_default_params = {
"thinking": {
"type": "enabled",
"budget_tokens": 10000,
}
}
optional_params: dict = {}
result = config.map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params,
model="gemini-3-flash",
drop_params=False,
)
# Check that thinkingConfig was created with thinkingLevel
assert "thinkingConfig" in result, "Should have thinkingConfig in optional_params"
thinking_config = result["thinkingConfig"]
assert "thinkingLevel" in thinking_config, "Should have thinkingLevel for Gemini 3"
assert "thinkingBudget" not in thinking_config, "Should NOT have thinkingBudget for Gemini 3"
assert thinking_config["includeThoughts"] is True
# Test with Gemini 2 model
optional_params_2 = {}
result_2 = config.map_openai_params(
non_default_params=non_default_params,
optional_params=optional_params_2,
model="gemini-2.5-flash",
drop_params=False,
)
# Check that thinkingConfig was created with thinkingBudget
assert "thinkingConfig" in result_2, "Should have thinkingConfig in optional_params"
thinking_config_2 = result_2["thinkingConfig"]
assert "thinkingBudget" in thinking_config_2, "Should have thinkingBudget for Gemini 2"
assert "thinkingLevel" not in thinking_config_2, "Should NOT have thinkingLevel for Gemini 2"
assert thinking_config_2["includeThoughts"] is True
assert thinking_config_2["thinkingBudget"] == 10000
def test_gemini_image_size_limit_exceeded():
"""
Test that large images exceeding MAX_IMAGE_URL_DOWNLOAD_SIZE_MB are rejected.
This validates that the 50MB default limit prevents downloading very large images
that could cause memory issues and pod crashes.
"""
messages = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "What is in this image?"
},
{
"type": "image_url",
"image_url": "https://upload.wikimedia.org/wikipedia/commons/5/51/Blue_Marble_2002.jpg"
}
]
}
]
with pytest.raises(litellm.ImageFetchError) as excinfo:
completion(
model="gemini/gemini-2.5-flash-lite",
messages=messages
)
error_message = str(excinfo.value)
assert "Image size" in error_message
assert "exceeds maximum allowed size" in error_message

99
tests/test_litellm/litellm_core_utils/test_image_handling.py
View file

@ -1,7 +1,10 @@
from unittest.mock import patch
import pytest
from httpx import Request, Response
import litellm
from litellm import constants
from litellm.litellm_core_utils.prompt_templates.image_handling import (
convert_url_to_base64,
)
@ -39,3 +42,99 @@ def test_completion_with_invalid_image_url(monkeypatch):
)
assert excinfo.value.status_code == 400
assert "Unable to fetch image" in str(excinfo.value)
class LargeImageClient:
"""
Client that returns a large image exceeding size limit.
"""
def __init__(self, size_mb=100, include_content_length=True):
self.size_mb = size_mb
self.include_content_length = include_content_length
def get(self, url, follow_redirects=True):
size_bytes = int(self.size_mb * 1024 * 1024)
headers = {"Content-Type": "image/jpeg"}
if self.include_content_length:
headers["Content-Length"] = str(size_bytes)
return Response(
status_code=200,
headers=headers,
content=b"x" * size_bytes,
request=Request("GET", url),
)
def test_image_exceeds_size_limit_with_content_length(monkeypatch):
"""
Test that images exceeding MAX_IMAGE_URL_DOWNLOAD_SIZE_MB are rejected when Content-Length header is present.
"""
monkeypatch.setattr(litellm, "module_level_client", LargeImageClient(size_mb=100))
with pytest.raises(litellm.ImageFetchError) as excinfo:
convert_url_to_base64("https://example.com/large-image.jpg")
assert "exceeds maximum allowed size" in str(excinfo.value)
assert "100.00MB" in str(excinfo.value)
assert "50.0MB" in str(excinfo.value)
def test_image_exceeds_size_limit_without_content_length(monkeypatch):
"""
Test that images exceeding MAX_IMAGE_URL_DOWNLOAD_SIZE_MB are rejected even without Content-Length header.
"""
monkeypatch.setattr(
litellm, "module_level_client", LargeImageClient(size_mb=100, include_content_length=False)
)
with pytest.raises(litellm.ImageFetchError) as excinfo:
convert_url_to_base64("https://example.com/large-image.jpg")
assert "exceeds maximum allowed size" in str(excinfo.value)
class SmallImageClient:
"""
Client that returns a small valid image.
"""
def get(self, url, follow_redirects=True):
size_bytes = 1024
headers = {
"Content-Type": "image/jpeg",
"Content-Length": str(size_bytes),
}
return Response(
status_code=200,
headers=headers,
content=b"x" * size_bytes,
request=Request("GET", url),
)
def test_image_within_size_limit(monkeypatch):
"""
Test that images within size limit are processed successfully.
"""
monkeypatch.setattr(litellm, "module_level_client", SmallImageClient())
result = convert_url_to_base64("https://example.com/small-image.jpg")
assert result.startswith("data:image/jpeg;base64,")
def test_image_size_limit_disabled(monkeypatch):
"""
Test that setting MAX_IMAGE_URL_DOWNLOAD_SIZE_MB to 0 disables all image URL downloads.
"""
import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling
monkeypatch.setattr(litellm, "module_level_client", SmallImageClient())
monkeypatch.setattr(image_handling, "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 0)
with pytest.raises(litellm.ImageFetchError) as excinfo:
convert_url_to_base64("https://example.com/image.jpg")
assert "Image URL download is disabled" in str(excinfo.value)
assert "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0" in str(excinfo.value)