From de901bbf249b5db6060b2a18cd9e271544222132 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 13 May 2025 16:10:55 -0700 Subject: [PATCH] [Fix] Allow overriding all constants using a .env variable (#10803) * fix: bump: DEFAULT_MAX_RECURSE_DEPTH * fix: bump: DEFAULT_MAX_RECURSE_DEPTH * test: test_vertex_ai_complex_response_schema * fix: allow all constants to be overriden * fix: allow all numeric constants to be overriden with env vars * fix: remove dup DEFAULT_MAX_TOKENS in constants.py * document all constants env vars * docs - DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD --- docs/my-website/docs/proxy/config_settings.md | 100 ++++++ litellm/constants.py | 286 +++++++++++------- tests/litellm/test_constants.py | 60 ++++ 3 files changed, 342 insertions(+), 104 deletions(-) create mode 100644 tests/litellm/test_constants.py diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 01ff24da694..a77beb01e16 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -331,14 +331,19 @@ router_settings: | AZURE_PASSWORD | Password for Azure services, use in conjunction with AZURE_USERNAME for azure ad token with basic username/password workflow | AZURE_FEDERATED_TOKEN_FILE | File path to Azure federated token | AZURE_KEY_VAULT_URI | URI for Azure Key Vault +| AZURE_OPERATION_POLLING_TIMEOUT | Timeout in seconds for Azure operation polling | AZURE_STORAGE_ACCOUNT_KEY | The Azure Storage Account Key to use for Authentication to Azure Blob Storage logging | AZURE_STORAGE_ACCOUNT_NAME | Name of the Azure Storage Account to use for logging to Azure Blob Storage | AZURE_STORAGE_FILE_SYSTEM | Name of the Azure Storage File System to use for logging to Azure Blob Storage. (Typically the Container name) | AZURE_STORAGE_TENANT_ID | The Application Tenant ID to use for Authentication to Azure Blob Storage logging | AZURE_STORAGE_CLIENT_ID | The Application Client ID to use for Authentication to Azure Blob Storage logging | AZURE_STORAGE_CLIENT_SECRET | The Application Client Secret to use for Authentication to Azure Blob Storage logging +| BATCH_STATUS_POLL_INTERVAL_SECONDS | Interval in seconds for polling batch status. Default is 3600 (1 hour) +| BATCH_STATUS_POLL_MAX_ATTEMPTS | Maximum number of attempts for polling batch status. Default is 24 (for 24 hours) +| BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75 | BERRISPEND_ACCOUNT_ID | Account ID for BerriSpend service | BRAINTRUST_API_KEY | API key for Braintrust integration +| CACHED_STREAMING_CHUNK_DELAY | Delay in seconds for cached streaming chunks. Default is 0.02 | CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI | CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI | CONFIG_FILE_PATH | File path for configuration file @@ -352,6 +357,9 @@ router_settings: | DATABASE_USER | Username for database connection | DATABASE_USERNAME | Alias for database user | DATABRICKS_API_BASE | Base URL for Databricks API +| DAYS_IN_A_MONTH | Days in a month for calculation purposes. Default is 28 +| DAYS_IN_A_WEEK | Days in a week for calculation purposes. Default is 7 +| DAYS_IN_A_YEAR | Days in a year for calculation purposes. Default is 365 | DD_BASE_URL | Base URL for Datadog integration | DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration | _DATADOG_BASE_URL | (Alternative to DD_BASE_URL) Base URL for Datadog integration @@ -362,6 +370,38 @@ router_settings: | DD_SERVICE | Service identifier for Datadog logs. Defaults to "litellm-server" | DD_VERSION | Version identifier for Datadog logs. Defaults to "unknown" | DEBUG_OTEL | Enable debug mode for OpenTelemetry +| DEFAULT_ALLOWED_FAILS | Maximum failures allowed before cooling down a model. Default is 3 +| DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS | Default maximum tokens for Anthropic chat completions. Default is 4096 +| DEFAULT_BATCH_SIZE | Default batch size for operations. Default is 512 +| DEFAULT_COOLDOWN_TIME_SECONDS | Duration in seconds to cooldown a model after failures. Default is 5 +| DEFAULT_CRON_JOB_LOCK_TTL_SECONDS | Time-to-live for cron job locks in seconds. Default is 60 (1 minute) +| DEFAULT_FAILURE_THRESHOLD_PERCENT | Threshold percentage of failures to cool down a deployment. Default is 0.5 (50%) +| DEFAULT_FLUSH_INTERVAL_SECONDS | Default interval in seconds for flushing operations. Default is 5 +| DEFAULT_HEALTH_CHECK_INTERVAL | Default interval in seconds for health checks. Default is 300 (5 minutes) +| DEFAULT_IMAGE_HEIGHT | Default height for images. Default is 300 +| DEFAULT_IMAGE_TOKEN_COUNT | Default token count for images. Default is 250 +| DEFAULT_IMAGE_WIDTH | Default width for images. Default is 300 +| DEFAULT_IN_MEMORY_TTL | Default time-to-live for in-memory cache in seconds. Default is 5 +| DEFAULT_MAX_LRU_CACHE_SIZE | Default maximum size for LRU cache. Default is 16 +| DEFAULT_MAX_RECURSE_DEPTH | Default maximum recursion depth. Default is 100 +| DEFAULT_MAX_RETRIES | Default maximum retry attempts. Default is 2 +| DEFAULT_MAX_TOKENS | Default maximum tokens for LLM calls. Default is 4096 +| DEFAULT_MAX_TOKENS_FOR_TRITON | Default maximum tokens for Triton models. Default is 2000 +| DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT | Default token count for mock response completions. Default is 20 +| DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT | Default token count for mock response prompts. Default is 10 +| DEFAULT_MODEL_CREATED_AT_TIME | Default creation timestamp for models. Default is 1677610602 +| DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD | Default threshold for prompt injection similarity. Default is 0.7 +| DEFAULT_POLLING_INTERVAL | Default polling interval for schedulers in seconds. Default is 0.03 +| DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET | Default high reasoning effort thinking budget. Default is 4096 +| DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET | Default low reasoning effort thinking budget. Default is 1024 +| DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET | Default medium reasoning effort thinking budget. Default is 2048 +| DEFAULT_REDIS_SYNC_INTERVAL | Default Redis synchronization interval in seconds. Default is 1 +| DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND | Default price per second for Replicate GPU. Default is 0.001400 +| DEFAULT_REPLICATE_POLLING_DELAY_SECONDS | Default delay in seconds for Replicate polling. Default is 1 +| DEFAULT_REPLICATE_POLLING_RETRIES | Default number of retries for Replicate polling. Default is 5 +| DEFAULT_SLACK_ALERTING_THRESHOLD | Default threshold for Slack alerting. Default is 300 +| DEFAULT_SOFT_BUDGET | Default soft budget for LiteLLM proxy keys. Default is 50.0 +| DEFAULT_TRIM_RATIO | Default ratio of tokens to trim from prompt end. Default is 0.75 | DIRECT_URL | Direct URL for service endpoint | DISABLE_ADMIN_UI | Toggle to disable the admin UI | DISABLE_SCHEMA_UPDATE | Toggle to disable schema updates @@ -369,8 +409,19 @@ router_settings: | DOCS_FILTERED | Flag indicating filtered documentation | DOCS_TITLE | Title of the documentation pages | DOCS_URL | The path to the Swagger API documentation. **By default this is "/"** +| EMAIL_LOGO_URL | URL for the logo used in emails | EMAIL_SUPPORT_CONTACT | Support contact email address | EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING | Flag to enable new multi-instance rate limiting. **Default is False** +| FIREWORKS_AI_4_B | Size parameter for Fireworks AI 4B model. Default is 4 +| FIREWORKS_AI_16_B | Size parameter for Fireworks AI 16B model. Default is 16 +| FIREWORKS_AI_56_B_MOE | Size parameter for Fireworks AI 56B MOE model. Default is 56 +| FIREWORKS_AI_80_B | Size parameter for Fireworks AI 80B model. Default is 80 +| FIREWORKS_AI_176_B_MOE | Size parameter for Fireworks AI 176B MOE model. Default is 176 +| FUNCTION_DEFINITION_TOKEN_COUNT | Token count for function definitions. Default is 9 +| GALILEO_BASE_URL | Base URL for Galileo platform +| GALILEO_PASSWORD | Password for Galileo authentication +| GALILEO_PROJECT_ID | Project ID for Galileo usage +| GALILEO_USERNAME | Username for Galileo authentication | GCS_BUCKET_NAME | Name of the Google Cloud Storage bucket | GCS_PATH_SERVICE_ACCOUNT | Path to the Google Cloud service account JSON file | GCS_FLUSH_INTERVAL | Flush interval for GCS logging (in seconds). Specify how often you want a log to be sent to GCS. **Default is 20 seconds** @@ -402,6 +453,7 @@ router_settings: | GOOGLE_CLIENT_ID | Client ID for Google OAuth | GOOGLE_CLIENT_SECRET | Client secret for Google OAuth | GOOGLE_KMS_RESOURCE_NAME | Name of the resource in Google KMS +| HEALTH_CHECK_TIMEOUT_SECONDS | Timeout in seconds for health checks. Default is 60 | HF_API_BASE | Base URL for Hugging Face API | HCP_VAULT_ADDR | Address for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) | HCP_VAULT_CLIENT_CERT | Path to client certificate for [Hashicorp Vault Secret Manager](../secret.md#hashicorp-vault) @@ -411,9 +463,13 @@ router_settings: | HCP_VAULT_CERT_ROLE | Role for [Hashicorp Vault Secret Manager Auth](../secret.md#hashicorp-vault) | HELICONE_API_KEY | API key for Helicone service | HOSTNAME | Hostname for the server, this will be [emitted to `datadog` logs](https://docs.litellm.ai/docs/proxy/logging#datadog) +| HOURS_IN_A_DAY | Hours in a day for calculation purposes. Default is 24 | HUGGINGFACE_API_BASE | Base URL for Hugging Face API | HUGGINGFACE_API_KEY | API key for Hugging Face API +| HUMANLOOP_PROMPT_CACHE_TTL_SECONDS | Time-to-live in seconds for cached prompts in Humanloop. Default is 60 | IAM_TOKEN_DB_AUTH | IAM token for database authentication +| INITIAL_RETRY_DELAY | Initial delay in seconds for retrying requests. Default is 0.5 +| JITTER | Jitter factor for retry delay calculations. Default is 0.75 | JSON_LOGS | Enable JSON formatted logging | JWT_AUDIENCE | Expected audience for JWT tokens | JWT_PUBLIC_KEY_URL | URL to fetch public key for JWT verification @@ -434,6 +490,7 @@ router_settings: | LANGSMITH_PROJECT | Project name for Langsmith integration | LANGSMITH_SAMPLING_RATE | Sampling rate for Langsmith logging | LANGTRACE_API_KEY | API key for Langtrace service +| LENGTH_OF_LITELLM_GENERATED_KEY | Length of keys generated by LiteLLM. Default is 16 | LITERAL_API_KEY | API key for Literal integration | LITERAL_API_URL | API URL for Literal service | LITERAL_BATCH_SIZE | Batch size for Literal operations @@ -454,6 +511,21 @@ router_settings: | LITELLM_TOKEN | Access token for LiteLLM integration | LITELLM_PRINT_STANDARD_LOGGING_PAYLOAD | If true, prints the standard logging payload to the console - useful for debugging | LOGFIRE_TOKEN | Token for Logfire logging service +| MAX_EXCEPTION_MESSAGE_LENGTH | Maximum length for exception messages. Default is 2000 +| MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000 +| MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the long side of high-resolution images. Default is 2000 +| MAX_REDIS_BUFFER_DEQUEUE_COUNT | Maximum count for Redis buffer dequeue operations. Default is 100 +| MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the short side of high-resolution images. Default is 768 +| MAX_SIZE_IN_MEMORY_QUEUE | Maximum size for in-memory queue. Default is 10000 +| MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB | Maximum size in KB for each item in memory cache. Default is 512 or 1024 +| MAX_SPENDLOG_ROWS_TO_QUERY | Maximum number of spend log rows to query. Default is 1,000,000 +| MAX_TEAM_LIST_LIMIT | Maximum number of teams to list. Default is 20 +| MAX_TILE_HEIGHT | Maximum height for image tiles. Default is 512 +| MAX_TILE_WIDTH | Maximum width for image tiles. Default is 512 +| MAX_TOKEN_TRIMMING_ATTEMPTS | Maximum number of attempts to trim a token message. Default is 10 +| MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0 +| MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001 +| MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024 | MISTRAL_API_BASE | Base URL for Mistral API | MISTRAL_API_KEY | API key for Mistral API | MICROSOFT_CLIENT_ID | Client ID for Microsoft services @@ -462,10 +534,12 @@ router_settings: | MICROSOFT_SERVICE_PRINCIPAL_ID | Service Principal ID for Microsoft Enterprise Application. (This is an advanced feature if you want litellm to auto-assign members to Litellm Teams based on their Microsoft Entra ID Groups) | NO_DOCS | Flag to disable documentation generation | NO_PROXY | List of addresses to bypass proxy +| NON_LLM_CONNECTION_TIMEOUT | Timeout in seconds for non-LLM service connections. Default is 15 | OAUTH_TOKEN_INFO_ENDPOINT | Endpoint for OAuth token info retrieval | OPENAI_BASE_URL | Base URL for OpenAI API | OPENAI_API_BASE | Base URL for OpenAI API | OPENAI_API_KEY | API key for OpenAI services +| OPENAI_FILE_SEARCH_COST_PER_1K_CALLS | Cost per 1000 calls for OpenAI file search. Default is 0.0025 | OPENAI_ORGANIZATION | Organization identifier for OpenAI | OPENID_BASE_URL | Base URL for OpenID Connect services | OPENID_CLIENT_ID | Client ID for OpenID Connect authentication @@ -487,21 +561,37 @@ router_settings: | PREDIBASE_API_BASE | Base URL for Predibase API | PRESIDIO_ANALYZER_API_BASE | Base URL for Presidio Analyzer service | PRESIDIO_ANONYMIZER_API_BASE | Base URL for Presidio Anonymizer service +| PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES | Refresh interval in minutes for Prometheus budget metrics. Default is 5 +| PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS | Fallback time in hours for sending stats to Prometheus. Default is 9 | PROMETHEUS_URL | URL for Prometheus service | PROMPTLAYER_API_KEY | API key for PromptLayer integration | PROXY_ADMIN_ID | Admin identifier for proxy server | PROXY_BASE_URL | Base URL for proxy service +| PROXY_BATCH_WRITE_AT | Time in seconds to wait before batch writing spend logs to the database. Default is 10 +| PROXY_BUDGET_RESCHEDULER_MAX_TIME | Maximum time in seconds to wait before checking database for budget resets. Default is 605 +| PROXY_BUDGET_RESCHEDULER_MIN_TIME | Minimum time in seconds to wait before checking database for budget resets. Default is 597 | PROXY_LOGOUT_URL | URL for logging out of the proxy service | LITELLM_MASTER_KEY | Master key for proxy authentication | QDRANT_API_BASE | Base URL for Qdrant API | QDRANT_API_KEY | API key for Qdrant service +| QDRANT_SCALAR_QUANTILE | Scalar quantile for Qdrant operations. Default is 0.99 | QDRANT_URL | Connection URL for Qdrant database +| QDRANT_VECTOR_SIZE | Vector size for Qdrant operations. Default is 1536 +| REDIS_CONNECTION_POOL_TIMEOUT | Timeout in seconds for Redis connection pool. Default is 5 | REDIS_HOST | Hostname for Redis server | REDIS_PASSWORD | Password for Redis service | REDIS_PORT | Port number for Redis server +| REDIS_SOCKET_TIMEOUT | Timeout in seconds for Redis socket operations. Default is 0.1 | REDOC_URL | The path to the Redoc Fast API documentation. **By default this is "/redoc"** +| REPEATED_STREAMING_CHUNK_LIMIT | Limit for repeated streaming chunks to detect looping. Default is 100 +| REPLICATE_MODEL_NAME_WITH_ID_LENGTH | Length of Replicate model names with ID. Default is 64 +| REPLICATE_POLLING_DELAY_SECONDS | Delay in seconds for Replicate polling operations. Default is 0.5 +| REQUEST_TIMEOUT | Timeout in seconds for requests. Default is 6000 +| ROUTER_MAX_FALLBACKS | Maximum number of fallbacks for router. Default is 5 +| SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours) | SERVER_ROOT_PATH | Root path for the server application | SET_VERBOSE | Flag to enable verbose logging +| SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD | Minimum number of requests to consider "reasonable traffic" for single-deployment cooldown logic. Default is 1000 | SLACK_DAILY_REPORT_FREQUENCY | Frequency of daily Slack reports (e.g., daily, weekly) | SLACK_WEBHOOK_URL | Webhook URL for Slack integration | SMTP_HOST | Hostname for the SMTP server @@ -518,7 +608,17 @@ router_settings: | SUPABASE_KEY | API key for Supabase service | SUPABASE_URL | Base URL for Supabase instance | STORE_MODEL_IN_DB | If true, enables storing model + credential information in the DB. +| SYSTEM_MESSAGE_TOKEN_COUNT | Token count for system messages. Default is 4 | TEST_EMAIL_ADDRESS | Email address used for testing purposes +| TOGETHER_AI_4_B | Size parameter for Together AI 4B model. Default is 4 +| TOGETHER_AI_8_B | Size parameter for Together AI 8B model. Default is 8 +| TOGETHER_AI_21_B | Size parameter for Together AI 21B model. Default is 21 +| TOGETHER_AI_41_B | Size parameter for Together AI 41B model. Default is 41 +| TOGETHER_AI_80_B | Size parameter for Together AI 80B model. Default is 80 +| TOGETHER_AI_110_B | Size parameter for Together AI 110B model. Default is 110 +| TOGETHER_AI_EMBEDDING_150_M | Size parameter for Together AI 150M embedding model. Default is 150 +| TOGETHER_AI_EMBEDDING_350_M | Size parameter for Together AI 350M embedding model. Default is 350 +| TOOL_CHOICE_OBJECT_TOKEN_COUNT | Token count for tool choice objects. Default is 4 | UI_LOGO_PATH | Path to the logo image used in the UI | UI_PASSWORD | Password for accessing the UI | UI_USERNAME | Username for accessing the UI diff --git a/litellm/constants.py b/litellm/constants.py index 46171621a78..1ee2e9517f3 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1,30 +1,48 @@ +import os from typing import List, Literal -ROUTER_MAX_FALLBACKS = 5 -DEFAULT_BATCH_SIZE = 512 -DEFAULT_FLUSH_INTERVAL_SECONDS = 5 -DEFAULT_MAX_RETRIES = 2 -DEFAULT_MAX_RECURSE_DEPTH = 100 -DEFAULT_FAILURE_THRESHOLD_PERCENT = ( - 0.5 # default cooldown a deployment if 50% of requests fail in a given minute +ROUTER_MAX_FALLBACKS = int(os.getenv("ROUTER_MAX_FALLBACKS", 5)) +DEFAULT_BATCH_SIZE = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) +DEFAULT_FLUSH_INTERVAL_SECONDS = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) +DEFAULT_MAX_RETRIES = int(os.getenv("DEFAULT_MAX_RETRIES", 2)) +DEFAULT_MAX_RECURSE_DEPTH = int(os.getenv("DEFAULT_MAX_RECURSE_DEPTH", 100)) +DEFAULT_FAILURE_THRESHOLD_PERCENT = float( + os.getenv("DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5) +) # default cooldown a deployment if 50% of requests fail in a given minute +DEFAULT_MAX_TOKENS = int(os.getenv("DEFAULT_MAX_TOKENS", 4096)) +DEFAULT_ALLOWED_FAILS = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3)) +DEFAULT_REDIS_SYNC_INTERVAL = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1)) +DEFAULT_COOLDOWN_TIME_SECONDS = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5)) +DEFAULT_REPLICATE_POLLING_RETRIES = int( + os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5) ) -DEFAULT_MAX_TOKENS = 4096 -DEFAULT_ALLOWED_FAILS = 3 -DEFAULT_REDIS_SYNC_INTERVAL = 1 -DEFAULT_COOLDOWN_TIME_SECONDS = 5 -DEFAULT_REPLICATE_POLLING_RETRIES = 5 -DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = 1 -DEFAULT_IMAGE_TOKEN_COUNT = 250 -DEFAULT_IMAGE_WIDTH = 300 -DEFAULT_IMAGE_HEIGHT = 300 -DEFAULT_MAX_TOKENS = 256 # used when providers need a default -MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = 1024 # 1MB = 1024KB -SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD = 1000 # Minimum number of requests to consider "reasonable traffic". Used for single-deployment cooldown logic. +DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int( + os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1) +) +DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250)) +DEFAULT_IMAGE_WIDTH = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300)) +DEFAULT_IMAGE_HEIGHT = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300)) +MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int( + os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024) +) # 1MB = 1024KB +SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD = int( + os.getenv("SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD", 1000) +) # Minimum number of requests to consider "reasonable traffic". Used for single-deployment cooldown logic. + +DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = int( + os.getenv("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024) +) +DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET = int( + os.getenv("DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET", 2048) +) +DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = int( + os.getenv("DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET", 4096) +) +MAX_TOKEN_TRIMMING_ATTEMPTS = int( + os.getenv("MAX_TOKEN_TRIMMING_ATTEMPTS", 10) +) # Maximum number of attempts to trim the message + -DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = 1024 -DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET = 2048 -DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = 4096 -MAX_TOKEN_TRIMMING_ATTEMPTS = 10 # Maximum number of attempts to trim the message ########## Networking constants ############################################################## _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client for 1 hour @@ -33,70 +51,102 @@ REDIS_UPDATE_BUFFER_KEY = "litellm_spend_update_buffer" REDIS_DAILY_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_spend_update_buffer" REDIS_DAILY_TEAM_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_team_spend_update_buffer" REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY = "litellm_daily_tag_spend_update_buffer" -MAX_REDIS_BUFFER_DEQUEUE_COUNT = 100 -MAX_SIZE_IN_MEMORY_QUEUE = 10000 -MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = 1000 -############################################################################################### -MINIMUM_PROMPT_CACHE_TOKEN_COUNT = ( - 1024 # minimum number of tokens to cache a prompt by Anthropic +MAX_REDIS_BUFFER_DEQUEUE_COUNT = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)) +MAX_SIZE_IN_MEMORY_QUEUE = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", 10000)) +MAX_IN_MEMORY_QUEUE_FLUSH_COUNT = int( + os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000) +) +############################################################################################### +MINIMUM_PROMPT_CACHE_TOKEN_COUNT = int( + os.getenv("MINIMUM_PROMPT_CACHE_TOKEN_COUNT", 1024) +) # minimum number of tokens to cache a prompt by Anthropic +DEFAULT_TRIM_RATIO = float( + os.getenv("DEFAULT_TRIM_RATIO", 0.75) +) # default ratio of tokens to trim from the end of a prompt +HOURS_IN_A_DAY = int(os.getenv("HOURS_IN_A_DAY", 24)) +DAYS_IN_A_WEEK = int(os.getenv("DAYS_IN_A_WEEK", 7)) +DAYS_IN_A_MONTH = int(os.getenv("DAYS_IN_A_MONTH", 28)) +DAYS_IN_A_YEAR = int(os.getenv("DAYS_IN_A_YEAR", 365)) +REPLICATE_MODEL_NAME_WITH_ID_LENGTH = int( + os.getenv("REPLICATE_MODEL_NAME_WITH_ID_LENGTH", 64) ) -DEFAULT_TRIM_RATIO = 0.75 # default ratio of tokens to trim from the end of a prompt -HOURS_IN_A_DAY = 24 -DAYS_IN_A_WEEK = 7 -DAYS_IN_A_MONTH = 28 -DAYS_IN_A_YEAR = 365 -REPLICATE_MODEL_NAME_WITH_ID_LENGTH = 64 #### TOKEN COUNTING #### -FUNCTION_DEFINITION_TOKEN_COUNT = 9 -SYSTEM_MESSAGE_TOKEN_COUNT = 4 -TOOL_CHOICE_OBJECT_TOKEN_COUNT = 4 -DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT = 10 -DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT = 20 -MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES = 768 -MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES = 2000 -MAX_TILE_WIDTH = 512 -MAX_TILE_HEIGHT = 512 -OPENAI_FILE_SEARCH_COST_PER_1K_CALLS = 2.5 / 1000 -MIN_NON_ZERO_TEMPERATURE = 0.0001 +FUNCTION_DEFINITION_TOKEN_COUNT = int(os.getenv("FUNCTION_DEFINITION_TOKEN_COUNT", 9)) +SYSTEM_MESSAGE_TOKEN_COUNT = int(os.getenv("SYSTEM_MESSAGE_TOKEN_COUNT", 4)) +TOOL_CHOICE_OBJECT_TOKEN_COUNT = int(os.getenv("TOOL_CHOICE_OBJECT_TOKEN_COUNT", 4)) +DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT = int( + os.getenv("DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT", 10) +) +DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT = int( + os.getenv("DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", 20) +) +MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES = int( + os.getenv("MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES", 768) +) +MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES = int( + os.getenv("MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES", 2000) +) +MAX_TILE_WIDTH = int(os.getenv("MAX_TILE_WIDTH", 512)) +MAX_TILE_HEIGHT = int(os.getenv("MAX_TILE_HEIGHT", 512)) +OPENAI_FILE_SEARCH_COST_PER_1K_CALLS = float( + os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000) +) +MIN_NON_ZERO_TEMPERATURE = float(os.getenv("MIN_NON_ZERO_TEMPERATURE", 0.0001)) #### RELIABILITY #### -REPEATED_STREAMING_CHUNK_LIMIT = 100 # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives. -DEFAULT_MAX_LRU_CACHE_SIZE = 16 -INITIAL_RETRY_DELAY = 0.5 -MAX_RETRY_DELAY = 8.0 -JITTER = 0.75 -DEFAULT_IN_MEMORY_TTL = 5 # default time to live for the in-memory cache -DEFAULT_POLLING_INTERVAL = 0.03 # default polling interval for the scheduler -AZURE_OPERATION_POLLING_TIMEOUT = 120 -REDIS_SOCKET_TIMEOUT = 0.1 -REDIS_CONNECTION_POOL_TIMEOUT = 5 -NON_LLM_CONNECTION_TIMEOUT = 15 # timeout for adjacent services (e.g. jwt auth) -MAX_EXCEPTION_MESSAGE_LENGTH = 2000 -BEDROCK_MAX_POLICY_SIZE = 75 -REPLICATE_POLLING_DELAY_SECONDS = 0.5 -DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS = 4096 -TOGETHER_AI_4_B = 4 -TOGETHER_AI_8_B = 8 -TOGETHER_AI_21_B = 21 -TOGETHER_AI_41_B = 41 -TOGETHER_AI_80_B = 80 -TOGETHER_AI_110_B = 110 -TOGETHER_AI_EMBEDDING_150_M = 150 -TOGETHER_AI_EMBEDDING_350_M = 350 -QDRANT_SCALAR_QUANTILE = 0.99 -QDRANT_VECTOR_SIZE = 1536 -CACHED_STREAMING_CHUNK_DELAY = 0.02 -MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = 512 -DEFAULT_MAX_TOKENS_FOR_TRITON = 2000 +REPEATED_STREAMING_CHUNK_LIMIT = int( + os.getenv("REPEATED_STREAMING_CHUNK_LIMIT", 100) +) # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives. +DEFAULT_MAX_LRU_CACHE_SIZE = int(os.getenv("DEFAULT_MAX_LRU_CACHE_SIZE", 16)) +INITIAL_RETRY_DELAY = float(os.getenv("INITIAL_RETRY_DELAY", 0.5)) +MAX_RETRY_DELAY = float(os.getenv("MAX_RETRY_DELAY", 8.0)) +JITTER = float(os.getenv("JITTER", 0.75)) +DEFAULT_IN_MEMORY_TTL = int( + os.getenv("DEFAULT_IN_MEMORY_TTL", 5) +) # default time to live for the in-memory cache +DEFAULT_POLLING_INTERVAL = float( + os.getenv("DEFAULT_POLLING_INTERVAL", 0.03) +) # default polling interval for the scheduler +AZURE_OPERATION_POLLING_TIMEOUT = int(os.getenv("AZURE_OPERATION_POLLING_TIMEOUT", 120)) +REDIS_SOCKET_TIMEOUT = float(os.getenv("REDIS_SOCKET_TIMEOUT", 0.1)) +REDIS_CONNECTION_POOL_TIMEOUT = int(os.getenv("REDIS_CONNECTION_POOL_TIMEOUT", 5)) +NON_LLM_CONNECTION_TIMEOUT = int( + os.getenv("NON_LLM_CONNECTION_TIMEOUT", 15) +) # timeout for adjacent services (e.g. jwt auth) +MAX_EXCEPTION_MESSAGE_LENGTH = int(os.getenv("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)) +BEDROCK_MAX_POLICY_SIZE = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75)) +REPLICATE_POLLING_DELAY_SECONDS = float( + os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5) +) +DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS = int( + os.getenv("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096) +) +TOGETHER_AI_4_B = int(os.getenv("TOGETHER_AI_4_B", 4)) +TOGETHER_AI_8_B = int(os.getenv("TOGETHER_AI_8_B", 8)) +TOGETHER_AI_21_B = int(os.getenv("TOGETHER_AI_21_B", 21)) +TOGETHER_AI_41_B = int(os.getenv("TOGETHER_AI_41_B", 41)) +TOGETHER_AI_80_B = int(os.getenv("TOGETHER_AI_80_B", 80)) +TOGETHER_AI_110_B = int(os.getenv("TOGETHER_AI_110_B", 110)) +TOGETHER_AI_EMBEDDING_150_M = int(os.getenv("TOGETHER_AI_EMBEDDING_150_M", 150)) +TOGETHER_AI_EMBEDDING_350_M = int(os.getenv("TOGETHER_AI_EMBEDDING_350_M", 350)) +QDRANT_SCALAR_QUANTILE = float(os.getenv("QDRANT_SCALAR_QUANTILE", 0.99)) +QDRANT_VECTOR_SIZE = int(os.getenv("QDRANT_VECTOR_SIZE", 1536)) +CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0.02)) +MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int( + os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512) +) +DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000)) #### Networking settings #### -request_timeout: float = 6000 # time in seconds +request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds STREAM_SSE_DONE_STRING: str = "[DONE]" ### SPEND TRACKING ### -DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND = 0.001400 # price per second for a100 80GB -FIREWORKS_AI_56_B_MOE = 56 -FIREWORKS_AI_176_B_MOE = 176 -FIREWORKS_AI_4_B = 4 -FIREWORKS_AI_16_B = 16 -FIREWORKS_AI_80_B = 80 +DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND = float( + os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400) +) # price per second for a100 80GB +FIREWORKS_AI_56_B_MOE = int(os.getenv("FIREWORKS_AI_56_B_MOE", 56)) +FIREWORKS_AI_176_B_MOE = int(os.getenv("FIREWORKS_AI_176_B_MOE", 176)) +FIREWORKS_AI_4_B = int(os.getenv("FIREWORKS_AI_4_B", 4)) +FIREWORKS_AI_16_B = int(os.getenv("FIREWORKS_AI_16_B", 16)) +FIREWORKS_AI_80_B = int(os.getenv("FIREWORKS_AI_80_B", 80)) #### Logging callback constants #### REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM" @@ -497,22 +547,26 @@ known_tokenizer_config = { OPENAI_FINISH_REASONS = ["stop", "length", "function_call", "content_filter", "null"] -HUMANLOOP_PROMPT_CACHE_TTL_SECONDS = 60 # 1 minute +HUMANLOOP_PROMPT_CACHE_TTL_SECONDS = int( + os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) +) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION = "2019-07-07" -PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES = 5 +PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES = int( + os.getenv("PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES", 5) +) MCP_TOOL_NAME_PREFIX = "mcp_tool" ########################### LiteLLM Proxy Specific Constants ########################### ######################################################################################## -MAX_SPENDLOG_ROWS_TO_QUERY = ( - 1_000_000 # if spendLogs has more than 1M rows, do not query the DB -) -DEFAULT_SOFT_BUDGET = ( - 50.0 # by default all litellm proxy keys have a soft budget of 50.0 -) +MAX_SPENDLOG_ROWS_TO_QUERY = int( + os.getenv("MAX_SPENDLOG_ROWS_TO_QUERY", 1_000_000) +) # if spendLogs has more than 1M rows, do not query the DB +DEFAULT_SOFT_BUDGET = float( + os.getenv("DEFAULT_SOFT_BUDGET", 50.0) +) # by default all litellm proxy keys have a soft budget of 50.0 # makes it clear this is a rate limit error for a litellm virtual key RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY = "LiteLLM Virtual Key user_api_key_hash" @@ -527,10 +581,16 @@ BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES = [ "optimize-prompt/", ] -BATCH_STATUS_POLL_INTERVAL_SECONDS = 3600 # 1 hour -BATCH_STATUS_POLL_MAX_ATTEMPTS = 24 # for 24 hours +BATCH_STATUS_POLL_INTERVAL_SECONDS = int( + os.getenv("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600) +) # 1 hour +BATCH_STATUS_POLL_MAX_ATTEMPTS = int( + os.getenv("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24) +) # for 24 hours -HEALTH_CHECK_TIMEOUT_SECONDS = 60 # 60 seconds +HEALTH_CHECK_TIMEOUT_SECONDS = int( + os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60) +) # 60 seconds UI_SESSION_TOKEN_TEAM_ID = "litellm-dashboard" LITELLM_PROXY_ADMIN_NAME = "default_user_id" @@ -538,15 +598,33 @@ LITELLM_PROXY_ADMIN_NAME = "default_user_id" ########################### DB CRON JOB NAMES ########################### DB_SPEND_UPDATE_JOB_NAME = "db_spend_update_job" PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics_job" -DEFAULT_CRON_JOB_LOCK_TTL_SECONDS = 60 # 1 minute -PROXY_BUDGET_RESCHEDULER_MIN_TIME = 597 -PROXY_BUDGET_RESCHEDULER_MAX_TIME = 605 -PROXY_BATCH_WRITE_AT = 10 # in seconds -DEFAULT_HEALTH_CHECK_INTERVAL = 300 # 5 minutes -PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS = 9 -DEFAULT_MODEL_CREATED_AT_TIME = 1677610602 # returns on `/models` endpoint -DEFAULT_SLACK_ALERTING_THRESHOLD = 300 -MAX_TEAM_LIST_LIMIT = 20 -DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = 0.7 -LENGTH_OF_LITELLM_GENERATED_KEY = 16 -SECRET_MANAGER_REFRESH_INTERVAL = 86400 +DEFAULT_CRON_JOB_LOCK_TTL_SECONDS = int( + os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) +) # 1 minute +PROXY_BUDGET_RESCHEDULER_MIN_TIME = int( + os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597) +) +PROXY_BUDGET_RESCHEDULER_MAX_TIME = int( + os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605) +) +PROXY_BATCH_WRITE_AT = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds +DEFAULT_HEALTH_CHECK_INTERVAL = int( + os.getenv("DEFAULT_HEALTH_CHECK_INTERVAL", 300) +) # 5 minutes +PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS = int( + os.getenv("PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS", 9) +) +DEFAULT_MODEL_CREATED_AT_TIME = int( + os.getenv("DEFAULT_MODEL_CREATED_AT_TIME", 1677610602) +) # returns on `/models` endpoint +DEFAULT_SLACK_ALERTING_THRESHOLD = int( + os.getenv("DEFAULT_SLACK_ALERTING_THRESHOLD", 300) +) +MAX_TEAM_LIST_LIMIT = int(os.getenv("MAX_TEAM_LIST_LIMIT", 20)) +DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float( + os.getenv("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7) +) +LENGTH_OF_LITELLM_GENERATED_KEY = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16)) +SECRET_MANAGER_REFRESH_INTERVAL = int( + os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400) +) diff --git a/tests/litellm/test_constants.py b/tests/litellm/test_constants.py new file mode 100644 index 00000000000..8fea7342305 --- /dev/null +++ b/tests/litellm/test_constants.py @@ -0,0 +1,60 @@ +import inspect +import json +import os +import sys +from unittest import mock + +import httpx +import pytest +import respx +from fastapi.testclient import TestClient + +sys.path.insert(0, os.path.abspath("../..")) # + +import importlib + +import litellm +from litellm import constants + + +def test_all_numeric_constants_can_be_overridden(): + """ + Test that all integer and float constants in constants.py can be overridden with environment variables. + This ensures that any new constants added in the future will be configurable via environment variables. + """ + # Get all attributes from the constants module + constants_attributes = inspect.getmembers(constants) + + # Filter for uppercase constants (by convention) that are integers or floats + numeric_constants = [ + (name, value) + for name, value in constants_attributes + if name.isupper() and isinstance(value, (int, float)) + ] + + # Ensure we found some constants to test + assert len(numeric_constants) > 0, "No numeric constants found to test" + + print("all numeric constants", json.dumps(numeric_constants, indent=4)) + + # Verify all numeric constants have environment variable support + for name, value in numeric_constants: + # Skip constants that are not meant to be overridden (if any) + if name.startswith("_"): + continue + + # Create a test value that's different from the default + test_value = value + 1 if isinstance(value, int) else value + 0.1 + + # Set the environment variable + with mock.patch.dict(os.environ, {name: str(test_value)}): + print("overriding", name, "with", test_value) + importlib.reload(constants) + + # Get the new value after reload + new_value = getattr(constants, name) + + # Verify the value was overridden + assert ( + new_value == test_value + ), f"Failed to override {name} with environment variable. Expected {test_value}, got {new_value}"