diff --git a/litellm/constants.py b/litellm/constants.py index 9d18fe5489d..4ac6adaba73 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -4,7 +4,13 @@ from enum import Enum from types import MappingProxyType from typing import Final, Literal -from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_in_range, get_env_int_or_none +from litellm.litellm_core_utils.env_utils import ( + get_env_float, + get_env_float_or_none, + get_env_int, + get_env_int_in_range, + get_env_int_or_none, +) MICROSOFT_GRAPH_BETA_BASE: Final = "https://graph.microsoft.com/beta" OAUTH_TOKEN_EXCHANGE_CACHE_SAFETY_MARGIN_SECONDS: Final = 60 @@ -32,7 +38,7 @@ DEFER_PYDANTIC_BUILD: Final = os.getenv("DEFER_PYDANTIC_BUILD", "true") in ("tru DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"}) -ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5)) +ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5) ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000 RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( { @@ -68,10 +74,10 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset( "auto_router_capability_limit", } ) -DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) -DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) +DEFAULT_BATCH_SIZE: Final = get_env_int("DEFAULT_BATCH_SIZE", 512) +DEFAULT_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_FLUSH_INTERVAL_SECONDS", 5) CLICKHOUSE_BATCH_SIZE: Final = get_env_int("CLICKHOUSE_BATCH_SIZE", 10_000) -CLICKHOUSE_FLUSH_INTERVAL_SECONDS: Final = float(os.getenv("CLICKHOUSE_FLUSH_INTERVAL_SECONDS", "1.0")) +CLICKHOUSE_FLUSH_INTERVAL_SECONDS: Final = get_env_float("CLICKHOUSE_FLUSH_INTERVAL_SECONDS", 1.0) CLICKHOUSE_MAX_BUFFERED_ROWS: Final = get_env_int("CLICKHOUSE_MAX_BUFFERED_ROWS", 200_000) CLICKHOUSE_MAX_RETRIES: Final = get_env_int("CLICKHOUSE_MAX_RETRIES", 3) DEFAULT_CLICKHOUSE_DATABASE: Final = "litellm" @@ -89,9 +95,9 @@ LENS_DATASET_MAX_CASE_CHARS: Final = get_env_int("LENS_DATASET_MAX_CASE_CHARS", LENS_DATASET_TRACE_PAGE_SIZE: Final = 500 LENS_FEEDBACK_MAX_COMMENT_CHARS: Final = 10_000 LENS_FEEDBACK_MAX_SCORE: Final = 10 -DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) -DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) -DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = int(os.getenv("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", "16")) +DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10) +DEFAULT_S3_BATCH_SIZE: Final = get_env_int("DEFAULT_S3_BATCH_SIZE", 512) +DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = get_env_int("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", 16) DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY: Final = get_env_int("DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY", 200) S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS: Final = get_env_int_in_range("S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS", 30, 1, 3600) # https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html @@ -103,48 +109,43 @@ MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024 S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY" S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY" MAX_FILE_LIST_LIMIT: Final = 10000 -DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)) -DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)) +DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) +DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1) GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10 budget_reservation_disabled_info_emitted = False -DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1)) -DEFAULT_SQS_BATCH_SIZE: Final = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512)) +DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = get_env_int("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1) +DEFAULT_SQS_BATCH_SIZE: Final = get_env_int("DEFAULT_SQS_BATCH_SIZE", 512) SQS_SEND_MESSAGE_ACTION: Final = "SendMessage" SQS_API_VERSION: Final = "2012-11-05" -DEFAULT_MAX_RETRIES: Final = int(os.getenv("DEFAULT_MAX_RETRIES", 2)) +DEFAULT_MAX_RETRIES: Final = get_env_int("DEFAULT_MAX_RETRIES", 2) # Max records accepted in one POST /v1/callbacks/logs batch. Bounds the blast # radius: each record fans out to spend logs + every callback integration. MAX_CALLBACK_LOG_RECORDS: Final = 1000 -DEFAULT_MAX_RECURSE_DEPTH: Final = int(os.getenv("DEFAULT_MAX_RECURSE_DEPTH", 100)) +DEFAULT_MAX_RECURSE_DEPTH: Final = get_env_int("DEFAULT_MAX_RECURSE_DEPTH", 100) GUARDRAIL_ROTATION_ATTEMPTS: Final = 3 -DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER = int(os.getenv("DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER", 10)) -DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = float( - os.getenv("DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5) +DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER = get_env_int("DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER", 10) +DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float( + "DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5 ) # default cooldown a deployment if 50% of requests fail in a given minute -DEFAULT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_MAX_TOKENS", 4096)) -DEFAULT_ALLOWED_FAILS: Final = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3)) -DEFAULT_REDIS_SYNC_INTERVAL: Final = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1)) -DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5)) -DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = float( - os.getenv("DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS", "1") -) -DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5)) -DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)) -DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250)) +DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096) +DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3) +DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1) +DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5) +DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = get_env_float("DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS", 1.0) +DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLING_RETRIES", 5) +DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1) +DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250) HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0 # Maximum wall-clock seconds a streaming response is allowed to run. # Streams exceeding this duration are terminated with a Timeout error. # None (default) = no limit. Set env var to a number of seconds to enable globally. -_max_stream_duration_env: Final = os.getenv("LITELLM_MAX_STREAMING_DURATION_SECONDS", None) -LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = ( - float(_max_stream_duration_env) if _max_stream_duration_env is not None else None -) +LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = get_env_float_or_none("LITELLM_MAX_STREAMING_DURATION_SECONDS") # Maximum number of base64 characters to keep in logging payloads. # Data URIs exceeding this are replaced with a size placeholder. # Set to 0 to disable truncation. -MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64)) +MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64) BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024 REDACTED_BY_LITELLM: Final = "redacted-by-litellm" # in-memory stand-in handed to provider converters for redacted arguments; never stored @@ -158,44 +159,44 @@ MAX_BASE64_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_BASE64_LENGTH_STDOUT_LOG" LITELLM_DETAILED_TIMING: Final = os.getenv("LITELLM_DETAILED_TIMING", "false").lower() == "true" # Model cost map validation constants -MODEL_COST_MAP_MIN_MODEL_COUNT: Final = int( - os.getenv("MODEL_COST_MAP_MIN_MODEL_COUNT", 50) +MODEL_COST_MAP_MIN_MODEL_COUNT: Final = get_env_int( + "MODEL_COST_MAP_MIN_MODEL_COUNT", 50 ) # Minimum number of models a fetched cost map must contain to be considered valid -MODEL_COST_MAP_MAX_SHRINK_RATIO: Final = float( - os.getenv("MODEL_COST_MAP_MAX_SHRINK_RATIO", 0.5) +MODEL_COST_MAP_MAX_SHRINK_RATIO: Final = get_env_float( + "MODEL_COST_MAP_MAX_SHRINK_RATIO", 0.5 ) # Maximum allowed shrinkage ratio vs local backup (0.5 = reject if fetched map is <50% of backup) -DEFAULT_IMAGE_WIDTH: Final = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300)) -DEFAULT_IMAGE_HEIGHT: Final = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300)) +DEFAULT_IMAGE_WIDTH: Final = get_env_int("DEFAULT_IMAGE_WIDTH", 300) +DEFAULT_IMAGE_HEIGHT: Final = get_env_int("DEFAULT_IMAGE_HEIGHT", 300) # Maximum size for image URL downloads in MB (default 50MB, set to 0 to disable limit) # This prevents memory issues from downloading very large images # Maps to OpenAI's 50 MB payload limit - requests with images exceeding this size will be rejected # Set MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0 to disable image URL handling entirely -MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: Final = float(os.getenv("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 50)) -MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB: Final = int( - os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024) +MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: Final = get_env_float("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 50.0) +MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB: Final = get_env_int( + "MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024 ) # 1MB = 1024KB # Surrogate-repair fallback in _read_request_body runs two full-body re.sub passes # that block the event loop on multi-MB malformed bodies. Skip the repair above this # size and raise the existing 400 immediately. Set to 0 to disable the cap. MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB: Final = get_env_int("MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB", 1) -SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD: Final = int( - os.getenv("SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD", 1000) +SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD: Final = get_env_int( + "SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD", 1000 ) # Minimum number of requests to consider "reasonable traffic". Used for single-deployment cooldown logic. -DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS: Final = int( - os.getenv("DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS", 5) +DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS: Final = get_env_int( + "DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS", 5 ) # Minimum number of requests before applying error rate cooldown. Prevents cooldown from triggering on first failure. -DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET = int(os.getenv("DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET", 0)) +DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET = get_env_int("DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET", 0) # MCP Semantic Tool Filter Defaults DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL: Final = str( os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL", "text-embedding-3-small") ) -DEFAULT_MCP_SEMANTIC_FILTER_TOP_K: Final = int(os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_TOP_K", 10)) -DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = float( - os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3) +DEFAULT_MCP_SEMANTIC_FILTER_TOP_K: Final = get_env_int("DEFAULT_MCP_SEMANTIC_FILTER_TOP_K", 10) +DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = get_env_float( + "DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3 ) -MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH: Final = int(os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150)) +MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH: Final = get_env_int("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150) MAX_LITELLM_CALL_ID_LENGTH: Final = 256 MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH: Final = 2048 @@ -205,15 +206,15 @@ DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS: Final = 2000 DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL: Final = str( os.getenv("DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL", "text-embedding-3-small") ) -DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD", 0.75)) +DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD = get_env_float("DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD", 0.75) DEFAULT_OPENAI_MODERATIONS_MODEL: Final = "omni-moderation-latest" # MCP OAuth2 Client Credentials Defaults -MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS: Final = int(os.getenv("MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS", "60")) -MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE", "200")) -MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600")) -MCP_SSO_ASSERTION_CACHE_TTL_SECONDS: Final = int(os.getenv("MCP_SSO_ASSERTION_CACHE_TTL_SECONDS", "60")) +MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS", 60) +MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE: Final = get_env_int("MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE", 200) +MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL: Final = get_env_int("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", 3600) +MCP_SSO_ASSERTION_CACHE_TTL_SECONDS: Final = get_env_int("MCP_SSO_ASSERTION_CACHE_TTL_SECONDS", 60) # mcp_tool_permissions entry that grants every current and future tool on a server MCP_ALL_TOOLS_WILDCARD: Final = "*" @@ -222,20 +223,20 @@ MCP_ALL_TOOLS_WILDCARD: Final = "*" # npm/npx needs a writable cache dir; in containers the default (~/.npm) # may not exist or be read-only. /tmp is always writable. MCP_NPM_CACHE_DIR: Final = os.getenv("MCP_NPM_CACHE_DIR", "/tmp/.npm_mcp_cache") -MCP_OAUTH2_TOKEN_CACHE_MIN_TTL: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10")) +MCP_OAUTH2_TOKEN_CACHE_MIN_TTL: Final = get_env_int("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", 10) # Per-user OAuth token Redis cache (for server-side token storage) MCP_PER_USER_TOKEN_REDIS_KEY_PREFIX: Final = "mcp:per_user_token" -MCP_PER_USER_TOKEN_DEFAULT_TTL: Final = int( - os.getenv("MCP_PER_USER_TOKEN_DEFAULT_TTL", "43200") # 12 hours -) -MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = int(os.getenv("MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS", "60")) +MCP_PER_USER_TOKEN_DEFAULT_TTL: Final = get_env_int("MCP_PER_USER_TOKEN_DEFAULT_TTL", 43200) +MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS", 60) # MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers. -MCP_CLIENT_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0")) -MCP_TOOL_LISTING_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0")) -MCP_METADATA_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_METADATA_TIMEOUT", "10.0")) -MCP_HEALTH_CHECK_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", "10.0")) +MCP_CLIENT_TIMEOUT: Final = get_env_float("LITELLM_MCP_CLIENT_TIMEOUT", 60.0) +MCP_TOOL_LISTING_TIMEOUT: Final = get_env_float("LITELLM_MCP_TOOL_LISTING_TIMEOUT", 30.0) +MCP_METADATA_TIMEOUT: Final = get_env_float("LITELLM_MCP_METADATA_TIMEOUT", 10.0) +MCP_HEALTH_CHECK_TIMEOUT: Final = get_env_float("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", 10.0) +AGENT_HEALTH_CHECK_TIMEOUT_SECONDS: Final = get_env_float("LITELLM_AGENT_HEALTH_CHECK_TIMEOUT", 5.0) +AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS: Final = get_env_float("LITELLM_AGENT_HEALTH_CHECK_GATHER_TIMEOUT", 30.0) MCP_UPSTREAM_SESSION_SHUTDOWN_TIMEOUT_SECONDS: Final = 5.0 MCP_TOOL_LISTING_MAX_PAGES: Final = 1000 MCP_GATEWAY_SESSION_ID_PREFIX_LENGTH: Final = 8 @@ -253,7 +254,7 @@ MCP_STDIO_ALLOWED_COMMANDS: Final[frozenset] = frozenset( ) # MCP OAuth2 Token Exchange (OBO) Defaults -MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE: Final = int(os.getenv("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", "500")) +MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE: Final = get_env_int("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", 500) LITELLM_UI_ALLOW_HEADERS: Final = [ "x-litellm-semantic-filter", @@ -266,14 +267,14 @@ LITELLM_UI_ALLOW_HEADERS: Final = [ ] # Gemini model-specific minimal thinking budget constants -DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH: Final = int( - os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH", 1) +DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH: Final = get_env_int( + "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH", 1 ) -DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO: Final = int( - os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128) +DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO: Final = get_env_int( + "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128 ) -DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE: Final = int( - os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512) +DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE: Final = get_env_int( + "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512 ) # Maximum number of callbacks that can be registered @@ -295,8 +296,8 @@ PROXY_LLM_PROVIDER_FALLBACK: Final = "litellm_proxy" PROXY_REJECTED_BEFORE_ROUTING_KEY: Final = "proxy_rejected_before_routing" # Generic fallback for unknown models -DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int( - os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128) +DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = get_env_int( + "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128 ) # Provider-specific API base URLs @@ -315,19 +316,19 @@ OPEN_SANDBOX_DEFAULT_TIMEOUT: Final = 300 OPEN_SANDBOX_READY_TIMEOUT: Final = 30.0 OPEN_SANDBOX_POLL_INTERVAL: Final = 0.2 -DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = int(os.getenv("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024)) -DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET: Final = int( - os.getenv("DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET", 2048) +DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = get_env_int("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024) +DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET: Final = get_env_int( + "DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET", 2048 ) -DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = int(os.getenv("DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET", 4096)) -DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET = int(os.getenv("DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET", 8192)) -DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET = int(os.getenv("DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET", 16384)) -MAX_TOKEN_TRIMMING_ATTEMPTS: Final = int( - os.getenv("MAX_TOKEN_TRIMMING_ATTEMPTS", 10) +DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = get_env_int("DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET", 4096) +DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET = get_env_int("DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET", 8192) +DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET = get_env_int("DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET", 16384) +MAX_TOKEN_TRIMMING_ATTEMPTS: Final = get_env_int( + "MAX_TOKEN_TRIMMING_ATTEMPTS", 10 ) # Maximum number of attempts to trim the message RUNWAYML_DEFAULT_API_VERSION: Final = str(os.getenv("RUNWAYML_DEFAULT_API_VERSION", "2024-11-06")) -RUNWAYML_POLLING_TIMEOUT = int(os.getenv("RUNWAYML_POLLING_TIMEOUT", 600)) # 10 minutes default for image generation +RUNWAYML_POLLING_TIMEOUT = get_env_int("RUNWAYML_POLLING_TIMEOUT", 600) # 10 minutes default for image generation ########## Networking constants ############################################################## _DEFAULT_TTL_FOR_HTTPX_CLIENTS: Final = 3600 # 1 hour, re-use the same httpx client for 1 hour @@ -344,10 +345,10 @@ EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING: Final = 10_000 # Aiohttp connection pooling - prevents memory leaks from unbounded connection growth # Set to 0 for unlimited (not recommended for production) -AIOHTTP_CONNECTOR_LIMIT: Final = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000)) -AIOHTTP_CONNECTOR_LIMIT_PER_HOST: Final = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500)) -AIOHTTP_KEEPALIVE_TIMEOUT: Final = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 15)) -AIOHTTP_TTL_DNS_CACHE: Final = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300)) +AIOHTTP_CONNECTOR_LIMIT: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT", 1000) +AIOHTTP_CONNECTOR_LIMIT_PER_HOST: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500) +AIOHTTP_KEEPALIVE_TIMEOUT: Final = get_env_int("AIOHTTP_KEEPALIVE_TIMEOUT", 15) +AIOHTTP_TTL_DNS_CACHE: Final = get_env_int("AIOHTTP_TTL_DNS_CACHE", 300) # TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs # whose idle timeout is shorter than provider response timeouts (e.g. AWS NAT # Gateway: 350s vs OpenAI/Azure: 600s). Without this, the kernel sends nothing @@ -355,9 +356,9 @@ AIOHTTP_TTL_DNS_CACHE: Final = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300)) # arrives. Enabling SO_KEEPALIVE makes the kernel emit TCP probes that reset # the NAT idle timer. AIOHTTP_SO_KEEPALIVE: Final = os.getenv("AIOHTTP_SO_KEEPALIVE", "False").lower() == "true" -AIOHTTP_TCP_KEEPIDLE: Final = int(os.getenv("AIOHTTP_TCP_KEEPIDLE", 60)) -AIOHTTP_TCP_KEEPINTVL: Final = int(os.getenv("AIOHTTP_TCP_KEEPINTVL", 30)) -AIOHTTP_TCP_KEEPCNT: Final = int(os.getenv("AIOHTTP_TCP_KEEPCNT", 5)) +AIOHTTP_TCP_KEEPIDLE: Final = get_env_int("AIOHTTP_TCP_KEEPIDLE", 60) +AIOHTTP_TCP_KEEPINTVL: Final = get_env_int("AIOHTTP_TCP_KEEPINTVL", 30) +AIOHTTP_TCP_KEEPCNT: Final = get_env_int("AIOHTTP_TCP_KEEPCNT", 5) # enable_cleanup_closed is only needed for Python versions with the SSL leak bug # Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960) # Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78 @@ -370,10 +371,9 @@ AIOHTTP_NEEDS_CLEANUP_CLOSED: Final = (3, 13, 0) <= sys.version_info < ( # WebSocket constants # Default to None (unlimited) to match OpenAI's official agents SDK behavior # https://github.com/openai/openai-agents-python/blob/cf1b933660e44fd37b4350c41febab8221801409/src/agents/realtime/openai_realtime.py#L235 -_max_size_env: Final = os.getenv("REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES") -REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES: Final = int(_max_size_env) if _max_size_env is not None else None -REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = float( - os.getenv("REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", "20.0") +REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES: Final = get_env_int_or_none("REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES") +REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float( + "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0 ) # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code @@ -439,15 +439,16 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_sp REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer" REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer" REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer" -MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)) +MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = get_env_int("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100) REDIS_SPEND_LOGS_BUFFER_KEY: Final = "litellm_spend_logs_buffer" REDIS_SPEND_LOGS_BUFFER_MAX_ROWS: Final = 100000 REDIS_SPEND_LOGS_BUFFER_DEQUEUE_COUNT: Final = 1000 # Bounds asyncio.Queue() instances (log queues, spend update queues, etc.) to prevent unbounded memory growth -LITELLM_ASYNCIO_QUEUE_MAXSIZE: Final = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000)) -TOOL_POLICY_CACHE_TTL_SECONDS: Final = int(os.getenv("TOOL_POLICY_CACHE_TTL_SECONDS", 60)) -GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS: Final = int( - os.getenv("GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS", 24 * 60 * 60) +LITELLM_ASYNCIO_QUEUE_MAXSIZE: Final = get_env_int("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000) +TOOL_POLICY_CACHE_TTL_SECONDS: Final = get_env_int("TOOL_POLICY_CACHE_TTL_SECONDS", 60) +LITELLM_CONFIG_CACHE_TTL_SECONDS: Final = get_env_int("LITELLM_CONFIG_PARAM_CACHE_TTL_SECONDS", 60) +GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS: Final = get_env_int( + "GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS", 24 * 60 * 60 ) BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS: Final = 25_000 CONTENT_FILTER_STREAMING_HOLDBACK_CHARS: Final = 50 @@ -457,8 +458,8 @@ PRESIDIO_ANALYZE_CHUNK_OVERLAP_CHARS: Final = 4096 PRESIDIO_ANALYZE_CHUNK_CONCURRENCY: Final = 8 # Aggregation threshold: default to 80% of the asyncio queue maxsize so the check can always trigger. # Must be < LITELLM_ASYNCIO_QUEUE_MAXSIZE; if set higher the aggregation logic will never fire. -MAX_SIZE_IN_MEMORY_QUEUE: Final = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8))) -MAX_IN_MEMORY_QUEUE_FLUSH_COUNT: Final = int(os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000)) +MAX_SIZE_IN_MEMORY_QUEUE: Final = get_env_int("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8)) +MAX_IN_MEMORY_QUEUE_FLUSH_COUNT: Final = get_env_int("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000) ############################################################################################### # Providers will not cache a prefix below a minimum size. That minimum is per-model, not global: # Anthropic's ranges from 512 to 4096 depending on the model, and can differ per platform for the @@ -473,22 +474,22 @@ MINIMUM_PROMPT_CACHE_TOKEN_COUNT: Final = ( else DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT ) PROMPT_CACHE_LOOKBACK_POSITIONS: Final = 20 -DEFAULT_TRIM_RATIO: Final = float( - os.getenv("DEFAULT_TRIM_RATIO", 0.75) +DEFAULT_TRIM_RATIO: Final = get_env_float( + "DEFAULT_TRIM_RATIO", 0.75 ) # default ratio of tokens to trim from the end of a prompt -HOURS_IN_A_DAY: Final = int(os.getenv("HOURS_IN_A_DAY", 24)) -DAYS_IN_A_WEEK: Final = int(os.getenv("DAYS_IN_A_WEEK", 7)) -DAYS_IN_A_MONTH: Final = int(os.getenv("DAYS_IN_A_MONTH", 28)) -DAYS_IN_A_YEAR: Final = int(os.getenv("DAYS_IN_A_YEAR", 365)) -REPLICATE_MODEL_NAME_WITH_ID_LENGTH: Final = int(os.getenv("REPLICATE_MODEL_NAME_WITH_ID_LENGTH", 64)) +HOURS_IN_A_DAY: Final = get_env_int("HOURS_IN_A_DAY", 24) +DAYS_IN_A_WEEK: Final = get_env_int("DAYS_IN_A_WEEK", 7) +DAYS_IN_A_MONTH: Final = get_env_int("DAYS_IN_A_MONTH", 28) +DAYS_IN_A_YEAR: Final = get_env_int("DAYS_IN_A_YEAR", 365) +REPLICATE_MODEL_NAME_WITH_ID_LENGTH: Final = get_env_int("REPLICATE_MODEL_NAME_WITH_ID_LENGTH", 64) #### TOKEN COUNTING #### -FUNCTION_DEFINITION_TOKEN_COUNT: Final = int(os.getenv("FUNCTION_DEFINITION_TOKEN_COUNT", 9)) -SYSTEM_MESSAGE_TOKEN_COUNT: Final = int(os.getenv("SYSTEM_MESSAGE_TOKEN_COUNT", 4)) -TOOL_CHOICE_OBJECT_TOKEN_COUNT: Final = int(os.getenv("TOOL_CHOICE_OBJECT_TOKEN_COUNT", 4)) -DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT", 10)) -DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", 20)) -MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES: Final = int(os.getenv("MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES", 768)) -MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES: Final = int(os.getenv("MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES", 2000)) +FUNCTION_DEFINITION_TOKEN_COUNT: Final = get_env_int("FUNCTION_DEFINITION_TOKEN_COUNT", 9) +SYSTEM_MESSAGE_TOKEN_COUNT: Final = get_env_int("SYSTEM_MESSAGE_TOKEN_COUNT", 4) +TOOL_CHOICE_OBJECT_TOKEN_COUNT: Final = get_env_int("TOOL_CHOICE_OBJECT_TOKEN_COUNT", 4) +DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT: Final = get_env_int("DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT", 10) +DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT: Final = get_env_int("DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", 20) +MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES: Final = get_env_int("MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES", 768) +MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES: Final = get_env_int("MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES", 2000) # tiktoken's BPE merge loop is quadratic in the length of a single regex piece, so a long run of one # repeated character (dot leaders, whitespace, zero-padded base64) can take minutes on a multi-MB payload. # Encoding in chunks makes the cost linear, at a drift of at most ~1 token per chunk boundary. The upper @@ -512,69 +513,63 @@ TOKEN_COUNTER_MAX_CONCURRENT_COUNTS: Final = get_env_int_in_range( minimum=1, maximum=256, ) -MAX_TILE_WIDTH: Final = int(os.getenv("MAX_TILE_WIDTH", 512)) -MAX_TILE_HEIGHT: Final = int(os.getenv("MAX_TILE_HEIGHT", 512)) -OPENAI_FILE_SEARCH_COST_PER_1K_CALLS: Final = float(os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000)) +MAX_TILE_WIDTH: Final = get_env_int("MAX_TILE_WIDTH", 512) +MAX_TILE_HEIGHT: Final = get_env_int("MAX_TILE_HEIGHT", 512) +OPENAI_FILE_SEARCH_COST_PER_1K_CALLS: Final = get_env_float("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000) GROQ_BROWSER_VISIT_WEBSITE_COST_PER_CALL: Final = 1.0 / 1000 # Azure OpenAI Assistants feature costs # Source: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ -AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY: Final = float( - os.getenv("AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY", 0.1) # $0.1 USD per 1 GB/Day +AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY: Final = get_env_float("AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY", 0.1) +AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS: Final = get_env_float("AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS", 3.0) +AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS: Final = get_env_float( + "AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS", 12.0 ) -AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS: Final = float( - os.getenv("AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS", 3.0) # $0.003 USD per 1K Tokens -) -AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS: Final = float( - os.getenv("AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS", 12.0) # $0.012 USD per 1K Tokens -) -AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY: Final = float( - os.getenv("AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY", 0.1) # $0.1 USD per 1 GB/Day (same as file search) -) -MIN_NON_ZERO_TEMPERATURE: Final = float(os.getenv("MIN_NON_ZERO_TEMPERATURE", 0.0001)) +AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY: Final = get_env_float("AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY", 0.1) +MIN_NON_ZERO_TEMPERATURE: Final = get_env_float("MIN_NON_ZERO_TEMPERATURE", 0.0001) #### RELIABILITY #### -REPEATED_STREAMING_CHUNK_LIMIT: Final = int( - os.getenv("REPEATED_STREAMING_CHUNK_LIMIT", 100) +REPEATED_STREAMING_CHUNK_LIMIT: Final = get_env_int( + "REPEATED_STREAMING_CHUNK_LIMIT", 100 ) # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives. # Shared maxsize for functools.lru_cache usage across hot paths. # Defaulted to 64 to avoid cache thrash in multi-model production workloads. -DEFAULT_MAX_LRU_CACHE_SIZE: Final = int(os.getenv("DEFAULT_MAX_LRU_CACHE_SIZE", 64)) +DEFAULT_MAX_LRU_CACHE_SIZE: Final = get_env_int("DEFAULT_MAX_LRU_CACHE_SIZE", 64) _REALTIME_BODY_CACHE_SIZE = 1000 # Keep realtime helper caches bounded; workloads rarely exceed 1k models/intents -INITIAL_RETRY_DELAY: Final = float(os.getenv("INITIAL_RETRY_DELAY", 0.5)) -MAX_RETRY_DELAY: Final = float(os.getenv("MAX_RETRY_DELAY", 8.0)) -JITTER: Final = float(os.getenv("JITTER", 0.75)) -DEFAULT_IN_MEMORY_TTL = int(os.getenv("DEFAULT_IN_MEMORY_TTL", 5)) # default time to live for the in-memory cache -DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE: Final = int( - os.getenv("DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE", 1000) +INITIAL_RETRY_DELAY: Final = get_env_float("INITIAL_RETRY_DELAY", 0.5) +MAX_RETRY_DELAY: Final = get_env_float("MAX_RETRY_DELAY", 8.0) +JITTER: Final = get_env_float("JITTER", 0.75) +DEFAULT_IN_MEMORY_TTL = get_env_int("DEFAULT_IN_MEMORY_TTL", 5) # default time to live for the in-memory cache +DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE: Final = get_env_int( + "DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE", 1000 ) # default max size for redis batch cache -DEFAULT_POLLING_INTERVAL: Final = float( - os.getenv("DEFAULT_POLLING_INTERVAL", 0.03) +DEFAULT_POLLING_INTERVAL: Final = get_env_float( + "DEFAULT_POLLING_INTERVAL", 0.03 ) # default polling interval for the scheduler -AZURE_OPERATION_POLLING_TIMEOUT: Final = int(os.getenv("AZURE_OPERATION_POLLING_TIMEOUT", 120)) +AZURE_OPERATION_POLLING_TIMEOUT: Final = get_env_int("AZURE_OPERATION_POLLING_TIMEOUT", 120) AZURE_DOCUMENT_INTELLIGENCE_API_VERSION: Final = str(os.getenv("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "2024-11-30")) -AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: Final = int(os.getenv("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", 96)) -REDIS_SOCKET_TIMEOUT: Final = float(os.getenv("REDIS_SOCKET_TIMEOUT", 0.1)) +AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: Final = get_env_int("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", 96) +REDIS_SOCKET_TIMEOUT: Final = get_env_float("REDIS_SOCKET_TIMEOUT", 0.1) CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS: Final[float] = 5.0 -REDIS_CONNECTION_POOL_TIMEOUT: Final = int(os.getenv("REDIS_CONNECTION_POOL_TIMEOUT", 5)) -REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5)) -REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60)) +REDIS_CONNECTION_POOL_TIMEOUT: Final = get_env_int("REDIS_CONNECTION_POOL_TIMEOUT", 5) +REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD: Final = get_env_int("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5) +REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT: Final = get_env_int("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60) REDIS_CIRCUIT_BREAKER_ENABLED: Final = os.getenv("REDIS_CIRCUIT_BREAKER_ENABLED", "true").lower() == "true" # minimum seconds a timeout-only failure streak must span before it can open the breaker, # so one event-loop stall timing out many queued calls at once does not trip it -REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION: Final = float(os.getenv("REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", 5.0)) -REDIS_TIMEOUT_LOG_INTERVAL: Final = float(os.getenv("REDIS_TIMEOUT_LOG_INTERVAL", "5.0")) +REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION: Final = get_env_float("REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", 5.0) +REDIS_TIMEOUT_LOG_INTERVAL: Final = get_env_float("REDIS_TIMEOUT_LOG_INTERVAL", 5.0) # Seconds of idle before a Redis cluster connection is validated with a PING and # reconnected if dead, so a connection silently dropped by a cluster restart # (e.g. ElastiCache Serverless maintenance) is not reused while broken REDIS_CLUSTER_HEALTH_CHECK_INTERVAL: Final = 25 # Default Redis major version to assume when version cannot be determined # Using 7 as it's the modern version that supports LPOP with count parameter -DEFAULT_REDIS_MAJOR_VERSION: Final = int(os.getenv("DEFAULT_REDIS_MAJOR_VERSION", 7)) -NON_LLM_CONNECTION_TIMEOUT: Final = int( - os.getenv("NON_LLM_CONNECTION_TIMEOUT", 15) +DEFAULT_REDIS_MAJOR_VERSION: Final = get_env_int("DEFAULT_REDIS_MAJOR_VERSION", 7) +NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( + "NON_LLM_CONNECTION_TIMEOUT", 15 ) # timeout for adjacent services (e.g. jwt auth) -MAX_EXCEPTION_MESSAGE_LENGTH: Final = int(os.getenv("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)) -MAX_STRING_LENGTH_PROMPT_IN_DB: Final = int(os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)) -BEDROCK_MAX_POLICY_SIZE: Final = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75)) +MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) +MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) +BEDROCK_MAX_POLICY_SIZE: Final = get_env_int("BEDROCK_MAX_POLICY_SIZE", 75) # One entry per distinct AWS credential-argument set. Per-user cost attribution passes the attributed # identity as aws_session_name, so this bounds how many attributed identities keep a cached STS session. BEDROCK_IAM_CACHE_MAX_ENTRIES: Final = 1000 @@ -584,27 +579,27 @@ BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES: Final = 64 # Retire a cached STS credential this many seconds before AWS expires it, so a request that reads it # still has a usable credential for the whole call. STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 -BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = int(os.getenv("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)) +BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 -REPLICATE_POLLING_DELAY_SECONDS: Final = float(os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5)) -DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096)) +REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) +DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 -TOGETHER_AI_4_B: Final = int(os.getenv("TOGETHER_AI_4_B", 4)) -TOGETHER_AI_8_B: Final = int(os.getenv("TOGETHER_AI_8_B", 8)) -TOGETHER_AI_21_B: Final = int(os.getenv("TOGETHER_AI_21_B", 21)) -TOGETHER_AI_41_B: Final = int(os.getenv("TOGETHER_AI_41_B", 41)) -TOGETHER_AI_80_B: Final = int(os.getenv("TOGETHER_AI_80_B", 80)) -TOGETHER_AI_110_B: Final = int(os.getenv("TOGETHER_AI_110_B", 110)) -TOGETHER_AI_EMBEDDING_150_M: Final = int(os.getenv("TOGETHER_AI_EMBEDDING_150_M", 150)) -TOGETHER_AI_EMBEDDING_350_M: Final = int(os.getenv("TOGETHER_AI_EMBEDDING_350_M", 350)) -QDRANT_SCALAR_QUANTILE: Final = float(os.getenv("QDRANT_SCALAR_QUANTILE", 0.99)) -QDRANT_VECTOR_SIZE: Final = int(os.getenv("QDRANT_VECTOR_SIZE", 1536)) -CACHED_STREAMING_CHUNK_DELAY: Final = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0.02)) -AUDIO_SPEECH_CHUNK_SIZE: Final = int( - os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192) +TOGETHER_AI_4_B: Final = get_env_int("TOGETHER_AI_4_B", 4) +TOGETHER_AI_8_B: Final = get_env_int("TOGETHER_AI_8_B", 8) +TOGETHER_AI_21_B: Final = get_env_int("TOGETHER_AI_21_B", 21) +TOGETHER_AI_41_B: Final = get_env_int("TOGETHER_AI_41_B", 41) +TOGETHER_AI_80_B: Final = get_env_int("TOGETHER_AI_80_B", 80) +TOGETHER_AI_110_B: Final = get_env_int("TOGETHER_AI_110_B", 110) +TOGETHER_AI_EMBEDDING_150_M: Final = get_env_int("TOGETHER_AI_EMBEDDING_150_M", 150) +TOGETHER_AI_EMBEDDING_350_M: Final = get_env_int("TOGETHER_AI_EMBEDDING_350_M", 350) +QDRANT_SCALAR_QUANTILE: Final = get_env_float("QDRANT_SCALAR_QUANTILE", 0.99) +QDRANT_VECTOR_SIZE: Final = get_env_int("QDRANT_VECTOR_SIZE", 1536) +CACHED_STREAMING_CHUNK_DELAY: Final = get_env_float("CACHED_STREAMING_CHUNK_DELAY", 0.02) +AUDIO_SPEECH_CHUNK_SIZE: Final = get_env_int( + "AUDIO_SPEECH_CHUNK_SIZE", 8192 ) # chunk_size for audio speech streaming. Balance between latency and memory usage -DEFAULT_MAX_TOKENS_FOR_TRITON: Final = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000)) +DEFAULT_MAX_TOKENS_FOR_TRITON: Final = get_env_int("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000) #### Networking settings #### # Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this # value so longer-running surfaces (Router `timeout or litellm.request_timeout`, @@ -617,12 +612,13 @@ DEFAULT_REQUEST_TIMEOUT_SECONDS: Final[float] = 6000.0 # deadline and connect handshake (see ``http_handler`` cached handler paths). COMPLETION_HTTP_FALLBACK_SECONDS: Final[float] = 600.0 HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: Final[float] = 5.0 -SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS: Final[float] = float( - os.getenv("SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS", "5.0") +SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS: Final[float] = get_env_float("SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS", 5.0) +_request_timeout_from_env: Final = get_env_float_or_none("REQUEST_TIMEOUT") +request_timeout: float = ( + DEFAULT_REQUEST_TIMEOUT_SECONDS if _request_timeout_from_env is None else _request_timeout_from_env ) -request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS)))) -request_timeout_explicitly_set: bool = "REQUEST_TIMEOUT" in os.environ -DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)) # 10 minutes +request_timeout_explicitly_set: bool = _request_timeout_from_env is not None +DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0) # 10 minutes AGENT_KILL_SWITCH_TIMEOUT_SECONDS: Final = 10.0 AGENT_KILL_SWITCH_RESPONSE_BODY_MAX_CHARS: Final = 2000 # Patterns that indicate a localhost/internal URL in A2A agent cards that should be @@ -646,20 +642,20 @@ STREAM_SSE_DATA_PREFIX: Final[str] = "data: " STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n' STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8") ### SPEND TRACKING ### -DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = float( - os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400) +DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float( + "DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400 ) # price per second for a100 80GB -FIREWORKS_AI_56_B_MOE: Final = int(os.getenv("FIREWORKS_AI_56_B_MOE", 56)) -FIREWORKS_AI_176_B_MOE: Final = int(os.getenv("FIREWORKS_AI_176_B_MOE", 176)) -FIREWORKS_AI_4_B: Final = int(os.getenv("FIREWORKS_AI_4_B", 4)) -FIREWORKS_AI_16_B: Final = int(os.getenv("FIREWORKS_AI_16_B", 16)) -FIREWORKS_AI_80_B: Final = int(os.getenv("FIREWORKS_AI_80_B", 80)) +FIREWORKS_AI_56_B_MOE: Final = get_env_int("FIREWORKS_AI_56_B_MOE", 56) +FIREWORKS_AI_176_B_MOE: Final = get_env_int("FIREWORKS_AI_176_B_MOE", 176) +FIREWORKS_AI_4_B: Final = get_env_int("FIREWORKS_AI_4_B", 4) +FIREWORKS_AI_16_B: Final = get_env_int("FIREWORKS_AI_16_B", 16) +FIREWORKS_AI_80_B: Final = get_env_int("FIREWORKS_AI_80_B", 80) # https://docs.fireworks.ai/guides/prompt-caching (accessed 2026-09-19): serverless cached prompt tokens # default to a 50% discount off the input rate FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO: Final = 0.5 #### Logging callback constants #### REDACTED_BY_LITELM_STRING: Final = "REDACTED_BY_LITELM" -MAX_LANGFUSE_INITIALIZED_CLIENTS: Final = int(os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50)) +MAX_LANGFUSE_INITIALIZED_CLIENTS: Final = get_env_int("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50) LANGFUSE_SHUTDOWN_FLUSH_TIMEOUT_MILLIS: Final = 10_000 # Backpressure + lifetime bounds for the /v1/messages streaming relay (see # BaseAnthropicMessagesStreamingIterator.async_sse_wrapper). The relay queue is @@ -667,27 +663,25 @@ LANGFUSE_SHUTDOWN_FLUSH_TIMEOUT_MILLIS: Final = 10_000 # buffer the whole response in memory; the detached-drain cap bounds how many # post-disconnect drains may run concurrently so client behavior can't create # unbounded worker state. -ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE: Final = int( - os.getenv("ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", "1024") +ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE: Final = get_env_int( + "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 1024 ) # Setting this to 0 disables detached draining entirely: every post-disconnect # pump bills whatever partial output it has already collected and aborts the # upstream stream immediately, instead of continuing to drain for the real # terminal usage. -ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS: Final = int( - os.getenv("ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", "100") -) -LOGGING_WORKER_CONCURRENCY: Final = int(os.getenv("LOGGING_WORKER_CONCURRENCY", 100)) # Must be above 0 -LOGGING_WORKER_MAX_QUEUE_SIZE: Final = int(os.getenv("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000)) -LOGGING_WORKER_MAX_TIME_PER_COROUTINE: Final = float(os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0)) +ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS: Final = get_env_int("ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 100) +LOGGING_WORKER_CONCURRENCY: Final = get_env_int("LOGGING_WORKER_CONCURRENCY", 100) # Must be above 0 +LOGGING_WORKER_MAX_QUEUE_SIZE: Final = get_env_int("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000) +LOGGING_WORKER_MAX_TIME_PER_COROUTINE: Final = get_env_float("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0) LOGGING_WORKER_TIMEOUT_SUMMARY_WINDOW_SECONDS: Final = 5.0 -LOGGING_WORKER_CLEAR_PERCENTAGE: Final = int( - os.getenv("LOGGING_WORKER_CLEAR_PERCENTAGE", 50) +LOGGING_WORKER_CLEAR_PERCENTAGE: Final = get_env_int( + "LOGGING_WORKER_CLEAR_PERCENTAGE", 50 ) # Percentage of queue to clear (default: 50%) -MAX_ITERATIONS_TO_CLEAR_QUEUE: Final = int(os.getenv("MAX_ITERATIONS_TO_CLEAR_QUEUE", 200)) -MAX_TIME_TO_CLEAR_QUEUE: Final = float(os.getenv("MAX_TIME_TO_CLEAR_QUEUE", 5.0)) -LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS: Final = float( - os.getenv("LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS", 0.5) +MAX_ITERATIONS_TO_CLEAR_QUEUE: Final = get_env_int("MAX_ITERATIONS_TO_CLEAR_QUEUE", 200) +MAX_TIME_TO_CLEAR_QUEUE: Final = get_env_float("MAX_TIME_TO_CLEAR_QUEUE", 5.0) +LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS: Final = get_env_float( + "LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS", 0.5 ) # Cooldown time in seconds before allowing another aggressive clear (default: 0.5s) LOGGING_EXECUTOR_MAX_THREADS: Final = get_env_int("LOGGING_EXECUTOR_MAX_THREADS", 100) LOGGING_EXECUTOR_MAX_PENDING_TASKS: Final = get_env_int("LOGGING_EXECUTOR_MAX_PENDING_TASKS", 10_000) @@ -700,9 +694,9 @@ DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE: Final = os.getenv( LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED: Final = 499 -EMAIL_BUDGET_ALERT_TTL: Final = int(os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60)) # 24 hours in seconds -EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE: Final = float( - os.getenv("EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8) +EMAIL_BUDGET_ALERT_TTL: Final = get_env_int("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60) # 24 hours in seconds +EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE: Final = get_env_float( + "EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8 ) # 80% of max budget ############### LLM Provider Constants ############### ### ANTHROPIC CONSTANTS ### @@ -727,11 +721,11 @@ LITELLM_WEB_SEARCH_TOOL_NAME: Final = "litellm_web_search" DEFAULT_IMAGE_ENDPOINT_MODEL: Final = "dall-e-2" DEFAULT_VIDEO_ENDPOINT_MODEL: Final = "sora-2" -DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS: Final = int(os.getenv("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8)) +DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS: Final = get_env_int("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8) ### DATAFORSEO CONSTANTS ### -DEFAULT_DATAFORSEO_LOCATION_CODE: Final = int( - os.getenv("DEFAULT_DATAFORSEO_LOCATION_CODE", 2250) +DEFAULT_DATAFORSEO_LOCATION_CODE: Final = get_env_int( + "DEFAULT_DATAFORSEO_LOCATION_CODE", 2250 ) # Default to France (2250) - lower number, commonly used location LITELLM_CHAT_PROVIDERS: Final = [ @@ -1593,20 +1587,20 @@ OPENAI_FINISH_REASONS: Final = [ "tool_calls", "content_filter", ] -HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = int(os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60)) # 1 minute +HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = get_env_int("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" AZURE_STORAGE_DEFAULT_ENDPOINT_SUFFIX: Final = "core.windows.net" -PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES: Final = int( - os.getenv("PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES", 5) +PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES: Final = get_env_int( + "PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES", 5 ) PROMETHEUS_OVERFLOW_SERIES_LABEL_VALUE: Final = "other" PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX: Final = "litellm_admitted_series_" -CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = int(os.getenv("CLOUDZERO_EXPORT_INTERVAL_MINUTES", 60)) +CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = get_env_int("CLOUDZERO_EXPORT_INTERVAL_MINUTES", 60) MCP_TOOL_NAME_PREFIX: Final = "mcp_tool" -MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = int(os.getenv("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100)) +MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = get_env_int("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100) PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096 # Headers to control callbacks @@ -1644,11 +1638,11 @@ STANDARD_CUSTOMER_ID_HEADERS: Final = [ "x-litellm-customer-id", "x-litellm-end-user-id", ] -MAX_SPENDLOG_ROWS_TO_QUERY: Final = int( - os.getenv("MAX_SPENDLOG_ROWS_TO_QUERY", 1_000_000) +MAX_SPENDLOG_ROWS_TO_QUERY: Final = get_env_int( + "MAX_SPENDLOG_ROWS_TO_QUERY", 1_000_000 ) # if spendLogs has more than 1M rows, do not query the DB -DEFAULT_SOFT_BUDGET: Final = float( - os.getenv("DEFAULT_SOFT_BUDGET", 50.0) +DEFAULT_SOFT_BUDGET: Final = get_env_float( + "DEFAULT_SOFT_BUDGET", 50.0 ) # by default all litellm proxy keys have a soft budget of 50.0 # makes it clear this is a rate limit error for a litellm virtual key RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY: Final = "LiteLLM Virtual Key user_api_key_hash" @@ -1715,38 +1709,16 @@ TRANSCRIBE_MEDIA_FETCH_ATTEMPTS: Final = 3 TRANSCRIBE_MEDIA_LAST_MODIFIED_TOLERANCE_SECONDS: Final = 1.0 # S3 Last-Modified carries whole seconds only TRANSCRIBE_MEASURABLE_MEDIA_FORMATS: Final = frozenset({"flac", "mp3", "ogg", "wav"}) # what libsndfile can read -BATCH_STATUS_POLL_INTERVAL_SECONDS: Final = int(os.getenv("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600)) # 1 hour -BATCH_STATUS_POLL_MAX_ATTEMPTS: Final = int(os.getenv("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24)) # for 24 hours +BATCH_STATUS_POLL_INTERVAL_SECONDS: Final = get_env_int("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600) # 1 hour +BATCH_STATUS_POLL_MAX_ATTEMPTS: Final = get_env_int("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24) # for 24 hours BATCH_TPD_WINDOW_SECONDS: Final = 86400 BATCH_TPD_DESCRIPTOR_SUFFIX: Final = "_tpd" -HEALTH_CHECK_TIMEOUT_SECONDS: Final = int(os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)) # 60 seconds -_background_health_check_max_tokens_env: Final = os.getenv("BACKGROUND_HEALTH_CHECK_MAX_TOKENS") -try: - _raw_background_health_check_max_tokens: Final = ( - _background_health_check_max_tokens_env.strip() if _background_health_check_max_tokens_env is not None else "" - ) - BACKGROUND_HEALTH_CHECK_MAX_TOKENS: int | None = ( - int(_raw_background_health_check_max_tokens) if _raw_background_health_check_max_tokens else None - ) -except (ValueError, TypeError): - BACKGROUND_HEALTH_CHECK_MAX_TOKENS = None - - -_background_health_check_max_tokens_reasoning_env: Final = os.getenv("BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING") -try: - _raw_background_health_check_max_tokens_reasoning: Final = ( - _background_health_check_max_tokens_reasoning_env.strip() - if _background_health_check_max_tokens_reasoning_env is not None - else "" - ) - BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING: int | None = ( - int(_raw_background_health_check_max_tokens_reasoning) - if _raw_background_health_check_max_tokens_reasoning - else None - ) -except (ValueError, TypeError): - BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING = None +HEALTH_CHECK_TIMEOUT_SECONDS: Final = get_env_int("HEALTH_CHECK_TIMEOUT_SECONDS", 60) # 60 seconds +BACKGROUND_HEALTH_CHECK_MAX_TOKENS: int | None = get_env_int_or_none("BACKGROUND_HEALTH_CHECK_MAX_TOKENS") +BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING: int | None = get_env_int_or_none( + "BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING" +) LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME: Final = "litellm-internal-health-check" LITTELM_CLI_SERVICE_ACCOUNT_NAME: Final = "litellm-cli" @@ -1771,22 +1743,22 @@ OTEL_SERVICE_NAME_METADATA_KEYS: Final = ("otel_service_name_override", "otel_se # Key Rotation Constants LITELLM_KEY_ROTATION_ENABLED: Final = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false") -LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS: Final = int( - os.getenv("LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS", 86400) +LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS: Final = get_env_int( + "LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS", 86400 ) # 24 hours default LITELLM_KEY_ROTATION_GRACE_PERIOD: Final[str] = os.getenv( "LITELLM_KEY_ROTATION_GRACE_PERIOD", "" ) # Duration to keep old key valid after rotation (e.g. "24h", "2d"); empty = immediate revoke (default) -LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS: Final = int( - os.getenv("LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS", 600) +LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS: Final = get_env_int( + "LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS", 600 ) # 10 minutes default — caps the deadlock window if a pod crashes mid-rotation UI_SESSION_TOKEN_TEAM_ID: Final = "litellm-dashboard" LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED = os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED", "false") -LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS: Final = int( - os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS", 86400) +LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS: Final = get_env_int( + "LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS", 86400 ) # 24 hours default -LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE: Final = int( - os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE", 1000) +LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE: Final = get_env_int( + "LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE", 1000 ) LOGIN_THROTTLE_CACHE_KEY_PREFIX: Final = "login_fail" LOGIN_THROTTLE_UNKNOWN_SOURCE: Final = "unknown" @@ -1796,7 +1768,7 @@ LOGIN_THROTTLE_NOT_BLOCKED: Final = (0, 0) LITELLM_PROXY_ADMIN_NAME: Final = "default_user_id" LITELLM_PROXY_BUDGET_NAME: Final = "litellm-proxy-budget" GLOBAL_PROXY_SPEND_CACHE_KEY: Final = f"{LITELLM_PROXY_ADMIN_NAME}:spend" -LITELLM_EXECUTED_BATCH_CONCURRENCY: Final = max(1, int(os.getenv("LITELLM_EXECUTED_BATCH_CONCURRENCY", "4"))) +LITELLM_EXECUTED_BATCH_CONCURRENCY: Final = max(1, get_env_int("LITELLM_EXECUTED_BATCH_CONCURRENCY", 4)) ########################### CLI SSO AUTHENTICATION CONSTANTS ########################### LITELLM_CLI_SOURCE_IDENTIFIER: Final = "litellm-cli" @@ -1805,8 +1777,8 @@ CLI_SSO_SESSION_CACHE_KEY_PREFIX: Final = "cli_sso_session" CLI_SSO_SESSION_TTL_SECONDS: Final = 600 CLI_SESSION_KEY_PREFIX: Final = "cli-session" # Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility -CLI_JWT_EXPIRATION_HOURS: Final = int( - os.getenv("CLI_JWT_EXPIRATION_HOURS") or os.getenv("LITELLM_CLI_JWT_EXPIRATION_HOURS") or 24 +CLI_JWT_EXPIRATION_HOURS: Final = get_env_int( + "CLI_JWT_EXPIRATION_HOURS", get_env_int("LITELLM_CLI_JWT_EXPIRATION_HOURS", 24) ) # Comma-separated allowlisted OIDC claim map for CLI SSO polling, e.g. # "employment_type->acme_employment_type,org_info.department->department" @@ -1824,7 +1796,7 @@ DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME: Final = "db_daily_tag_spend_update_job" PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME: Final = "prometheus_emit_budget_metrics" CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data" MAVVRIK_FOCUS_EXPORT_JOB_NAME: Final = "mavvrik_focus_export_usage_data" -CLOUDZERO_MAX_FETCHED_DATA_RECORDS: Final = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)) +CLOUDZERO_MAX_FETCHED_DATA_RECORDS: Final = get_env_int("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000) SPEND_LOG_CLEANUP_JOB_NAME: Final = "spend_log_cleanup" BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME: Final = "background_health_check_db_save" KEY_ROTATION_JOB_NAME: Final = "litellm_key_rotation_job" @@ -1835,20 +1807,20 @@ USER_SPEND_ALERTS_JOB_ID: Final = "user_spend_alerts_job" PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job" SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report" SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning" -SPEND_LOG_RUN_LOOPS: Final = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500)) -SPEND_LOG_CLEANUP_BATCH_SIZE: Final = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000)) -SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)) -SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = float( - os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5) +SPEND_LOG_RUN_LOOPS: Final = get_env_int("SPEND_LOG_RUN_LOOPS", 500) +SPEND_LOG_CLEANUP_BATCH_SIZE: Final = get_env_int("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000) +SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = get_env_int("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3) +SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = get_env_float( + "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5 ) -SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", "300")) -SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", "30")) -SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = int(os.getenv("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", "100000")) -SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = float( - os.getenv("SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", "5") +SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", 300.0) +SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", 30.0) +SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = get_env_int("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", 100000) +SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = get_env_float( + "SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", 5.0 ) -SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS: Final = float( - os.getenv("SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS", "5") +SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS: Final = get_env_float( + "SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS", 5.0 ) TOOL_SPEND_TOP_TOOLS: Final = 100 MODEL_INSIGHTS_TOP_MODELS: Final = 10 @@ -1856,54 +1828,54 @@ MODEL_INSIGHTS_MAX_RANGE_DAYS: Final = 365 MODEL_INSIGHTS_DEFAULT_TASK: Final = "uncategorized" MODEL_INSIGHTS_TASK_TAG_PREFIX: Final = "task:" SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day") -SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7)) -SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000))) -SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_ROWS", "100"))) -SPEND_ROLLUP_LOCK_TIMEOUT_MS: Final = max(1, int(os.getenv("SPEND_ROLLUP_LOCK_TIMEOUT_MS", "5000"))) -SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100)) -SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000"))) -SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0)) -RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, int(os.getenv("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", "3"))) -RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = float(os.getenv("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", "0.2")) -SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000)) -PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, int(os.getenv("PROXY_DB_LOOKUP_MAX_CONCURRENCY", "25"))) -PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, float(os.getenv("PROXY_DB_LOOKUP_DEADLINE_SECONDS", "10"))) -PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, float(os.getenv("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", "30"))) -DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)) # 1 minute -PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)) -RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_BATCH_SIZE", "500"))) -RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN", "100"))) +SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = get_env_int("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7) +SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, get_env_int("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)) +SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, get_env_int("SPEND_LOG_WRITE_BATCH_MAX_ROWS", 100)) +SPEND_ROLLUP_LOCK_TIMEOUT_MS: Final = max(1, get_env_int("SPEND_ROLLUP_LOCK_TIMEOUT_MS", 5000)) +SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = get_env_int("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100) +SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, get_env_int("SPEND_LOG_QUEUE_MAX_BYTES", 64000000)) +SPEND_LOG_QUEUE_POLL_INTERVAL: Final = get_env_float("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0) +RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, get_env_int("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", 3)) +RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = get_env_float("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", 0.2) +SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000) +PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25)) +PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0)) +PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0)) +DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) # 1 minute +PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597) +RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500)) +RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN: Final = max(1, get_env_int("RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN", 100)) RESET_BUDGET_JOB_NAME: Final = "reset_budget_job" # Comfortably longer than one PROXY_BUDGET_RESCHEDULER_MIN_TIME tick, so a healthy # leader keeps the lease across its own run, and a crashed one strands the sweep for # at most a single tick. RESET_BUDGET_JOB_LOCK_TTL_SECONDS: Final[int] = 900 -PROXY_BATCH_POLLING_INTERVAL: Final = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 3600)) -MAX_OBJECTS_PER_POLL_CYCLE: Final = max(1, int(os.getenv("MAX_OBJECTS_PER_POLL_CYCLE", 50))) -MANAGED_OBJECT_STALENESS_CUTOFF_DAYS: Final = max(1, int(os.getenv("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7))) +PROXY_BATCH_POLLING_INTERVAL: Final = get_env_int("PROXY_BATCH_POLLING_INTERVAL", 3600) +MAX_OBJECTS_PER_POLL_CYCLE: Final = max(1, get_env_int("MAX_OBJECTS_PER_POLL_CYCLE", 50)) +MANAGED_OBJECT_STALENESS_CUTOFF_DAYS: Final = max(1, get_env_int("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7)) BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS: Final = 10.0 BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS: Final = 60 -STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, int(os.getenv("STALE_OBJECT_CLEANUP_BATCH_SIZE", 1000))) +STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, get_env_int("STALE_OBJECT_CLEANUP_BATCH_SIZE", 1000)) # Set PROXY_BATCH_POLLING_ENABLED=false to disable the CheckBatchCost and # CheckResponsesCost background polling jobs entirely (e.g. to avoid DB load on # installations with large numbers of stale managed objects). _batch_polling_env: Final = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower() PROXY_BATCH_POLLING_ENABLED: Final = _batch_polling_env == "true" -BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS: Final = float( - os.getenv("BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", "5") +BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS: Final = get_env_float( + "BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", 5.0 ) -BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = float( - os.getenv("BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", "60") +BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = get_env_float( + "BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", 60.0 ) -BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = float( - os.getenv("BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", "3600") +BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = get_env_float( + "BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", 3600.0 ) _background_interaction_cost_polling_env: Final = os.getenv( "BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true" ).lower() BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true" -PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)) -PROXY_BATCH_WRITE_AT: Final = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds, increased from 10 +PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605) +PROXY_BATCH_WRITE_AT: Final = get_env_int("PROXY_BATCH_WRITE_AT", 10) # in seconds, increased from 10 PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30) # APScheduler Configuration - MEMORY LEAK FIX @@ -1912,10 +1884,10 @@ APSCHEDULER_COALESCE: Final = os.getenv("APSCHEDULER_COALESCE", "True").lower() "true", "1", ] # collapse many missed runs into one -APSCHEDULER_MISFIRE_GRACE_TIME: Final = int( - os.getenv("APSCHEDULER_MISFIRE_GRACE_TIME", 3600) +APSCHEDULER_MISFIRE_GRACE_TIME: Final = get_env_int( + "APSCHEDULER_MISFIRE_GRACE_TIME", 3600 ) # ignore runs older than 1 hour (was 120) -APSCHEDULER_MAX_INSTANCES: Final = int(os.getenv("APSCHEDULER_MAX_INSTANCES", 1)) # prevent concurrent job instances +APSCHEDULER_MAX_INSTANCES: Final = get_env_int("APSCHEDULER_MAX_INSTANCES", 1) # prevent concurrent job instances APSCHEDULER_REPLACE_EXISTING: Final = os.getenv("APSCHEDULER_REPLACE_EXISTING", "True").lower() in [ "true", "1", @@ -1929,25 +1901,25 @@ DEFAULT_STAGGER_WINDOW_SECONDS: Final = 300 # This will run tag spcific tasks at a later time to smooth QPS DAILY_TAG_SPEND_BATCH_MULTIPLIER: Final = 2.3 -DEFAULT_HEALTH_CHECK_INTERVAL: Final = int(os.getenv("DEFAULT_HEALTH_CHECK_INTERVAL", 300)) # 5 minutes -DEFAULT_SHARED_HEALTH_CHECK_TTL: Final = int( - os.getenv("DEFAULT_SHARED_HEALTH_CHECK_TTL", 300) +DEFAULT_HEALTH_CHECK_INTERVAL: Final = get_env_int("DEFAULT_HEALTH_CHECK_INTERVAL", 300) # 5 minutes +DEFAULT_SHARED_HEALTH_CHECK_TTL: Final = get_env_int( + "DEFAULT_SHARED_HEALTH_CHECK_TTL", 300 ) # 5 minutes - TTL for cached health check results -DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL: Final = int( - os.getenv("DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL", 60) +DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL: Final = get_env_int( + "DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL", 60 ) # 1 minute - TTL for health check lock DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER: Final = 2 # health state is stale after interval * this -PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS: Final = int(os.getenv("PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS", 9)) -DEFAULT_MODEL_CREATED_AT_TIME: Final = int( - os.getenv("DEFAULT_MODEL_CREATED_AT_TIME", 1677610602) +PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS: Final = get_env_int("PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS", 9) +DEFAULT_MODEL_CREATED_AT_TIME: Final = get_env_int( + "DEFAULT_MODEL_CREATED_AT_TIME", 1677610602 ) # returns on `/models` endpoint -DEFAULT_SLACK_ALERTING_THRESHOLD: Final = int(os.getenv("DEFAULT_SLACK_ALERTING_THRESHOLD", 300)) -MAX_TEAM_LIST_LIMIT: Final = int(os.getenv("MAX_TEAM_LIST_LIMIT", 20)) -MAX_POLICY_ESTIMATE_IMPACT_ROWS: Final = int(os.getenv("MAX_POLICY_ESTIMATE_IMPACT_ROWS", 1000)) -DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7)) -LENGTH_OF_LITELLM_GENERATED_KEY: Final = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16)) -MINIMUM_CUSTOM_KEY_LENGTH: Final = int(os.getenv("MINIMUM_CUSTOM_KEY_LENGTH", 16)) -SECRET_MANAGER_REFRESH_INTERVAL: Final = int(os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400)) +DEFAULT_SLACK_ALERTING_THRESHOLD: Final = get_env_int("DEFAULT_SLACK_ALERTING_THRESHOLD", 300) +MAX_TEAM_LIST_LIMIT: Final = get_env_int("MAX_TEAM_LIST_LIMIT", 20) +MAX_POLICY_ESTIMATE_IMPACT_ROWS: Final = get_env_int("MAX_POLICY_ESTIMATE_IMPACT_ROWS", 1000) +DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = get_env_float("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7) +LENGTH_OF_LITELLM_GENERATED_KEY: Final = get_env_int("LENGTH_OF_LITELLM_GENERATED_KEY", 16) +MINIMUM_CUSTOM_KEY_LENGTH: Final = get_env_int("MINIMUM_CUSTOM_KEY_LENGTH", 16) +SECRET_MANAGER_REFRESH_INTERVAL: Final = get_env_int("SECRET_MANAGER_REFRESH_INTERVAL", 86400) OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS: Final = frozenset({"openai", "azure"}) LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "default_internal_user_params", @@ -1980,9 +1952,9 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "aws_sqs_callback_params", ] SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] -DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) +DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = get_env_int("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60) AUTH_OBJECTS_TARGET: Final = "auth_objects" -DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = int(os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)) +DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = get_env_int("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600) SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600 SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30 SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = 10000 @@ -2090,14 +2062,14 @@ SENTRY_PII_DENYLIST: Final = [ ] # CoroutineChecker cache configuration -COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY: Final = int(os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000)) +COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY: Final = get_env_int("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000) ########################### RAG Text Splitter Constants ########################### -DEFAULT_CHUNK_SIZE: Final = int(os.getenv("DEFAULT_CHUNK_SIZE", 1000)) -DEFAULT_CHUNK_OVERLAP: Final = int(os.getenv("DEFAULT_CHUNK_OVERLAP", 200)) +DEFAULT_CHUNK_SIZE: Final = get_env_int("DEFAULT_CHUNK_SIZE", 1000) +DEFAULT_CHUNK_OVERLAP: Final = get_env_int("DEFAULT_CHUNK_OVERLAP", 200) ########################### S3 Vectors RAG Constants ########################### -S3_VECTORS_DEFAULT_DIMENSION: Final = int(os.getenv("S3_VECTORS_DEFAULT_DIMENSION", 1024)) +S3_VECTORS_DEFAULT_DIMENSION: Final = get_env_int("S3_VECTORS_DEFAULT_DIMENSION", 1024) S3_VECTORS_DEFAULT_DISTANCE_METRIC: Final = str(os.getenv("S3_VECTORS_DEFAULT_DISTANCE_METRIC", "cosine")) S3_VECTORS_DEFAULT_NON_FILTERABLE_METADATA_KEYS: Final = ["source_text"] @@ -2110,11 +2082,11 @@ MICROSOFT_USER_LAST_NAME_ATTRIBUTE: Final = str(os.getenv("MICROSOFT_USER_LAST_N # Maximum payload size (in bytes) to fully serialize for DEBUG logging. # Payloads larger than this are truncated to avoid multi-second json.dumps blocking the response. -MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG: Final = int(os.getenv("MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG", 102400)) # 100 KB +MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG: Final = get_env_int("MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG", 102400) # 100 KB # Policy template enrichment -MAX_COMPETITOR_NAMES: Final = int(os.getenv("MAX_COMPETITOR_NAMES", 100)) -COMPETITOR_LLM_TEMPERATURE: Final = float(os.getenv("COMPETITOR_LLM_TEMPERATURE", 0.3)) +MAX_COMPETITOR_NAMES: Final = get_env_int("MAX_COMPETITOR_NAMES", 100) +COMPETITOR_LLM_TEMPERATURE: Final = get_env_float("COMPETITOR_LLM_TEMPERATURE", 0.3) DEFAULT_COMPETITOR_DISCOVERY_MODEL: Final = "gpt-4o-mini" # Advisor tool orchestration diff --git a/litellm/litellm_core_utils/env_utils.py b/litellm/litellm_core_utils/env_utils.py index d641884b4cd..6883af98bbd 100644 --- a/litellm/litellm_core_utils/env_utils.py +++ b/litellm/litellm_core_utils/env_utils.py @@ -4,23 +4,65 @@ Utility helpers for reading and parsing environment variables. import logging import os -from typing import Final +from typing import Final, TypeVar + +_NumberT: Final = TypeVar("_NumberT", int, float) + +# Marked before logging: the LiteLLM loggers' own filters read env knobs through these helpers, so +# warning first would re-enter the helper from inside the log call and recurse until RecursionError. +_reported_unparsable_values: Final[set[tuple[str, str]]] = set() # mutable-ok: process-wide warn-once registry + + +def _warn_unparsable_once(env_var: str, raw: str, number_type: type[int] | type[float]) -> None: + key: Final = (env_var, raw) + if key in _reported_unparsable_values: + return + _reported_unparsable_values.add(key) + logging.getLogger("LiteLLM").warning( + "%s=%r is not a valid %s. Treating it as unset.", env_var, raw, number_type.__name__ + ) + + +def _get_env_number_or_none(env_var: str, number_type: type[_NumberT]) -> _NumberT | None: + raw: Final = os.getenv(env_var) + if raw is None or not raw.strip(): + return None + try: + return number_type(raw.strip()) + except ValueError: + _warn_unparsable_once(env_var, raw, number_type) + return None + + +def get_env_int_or_none(env_var: str) -> int | None: + """Parse an environment variable as an integer, returning None when it is unset, blank or unparsable. + + Use this instead of `get_env_int` when callers must distinguish "explicitly configured" + from "left at the default", for example when an override should take precedence over a + value resolved from somewhere else. + """ + return _get_env_number_or_none(env_var, int) + + +def get_env_float_or_none(env_var: str) -> float | None: + """Float counterpart of `get_env_int_or_none`.""" + return _get_env_number_or_none(env_var, float) def get_env_int(env_var: str, default: int) -> int: - """Parse an environment variable as an integer, falling back to default on invalid values. + """Parse an environment variable as an integer, falling back to default when it is unset, blank or unparsable. - Handles empty strings, whitespace, and non-numeric values gracefully - so that misconfiguration doesn't crash the process at import time. + An unparsable value is warned about once, so misconfiguration neither crashes the process at + import time nor goes unnoticed. """ - raw = os.getenv(env_var) - if raw is None: - return default - raw = raw.strip() - try: - return int(raw) - except (ValueError, TypeError): - return default + value: Final = get_env_int_or_none(env_var) + return default if value is None else value + + +def get_env_float(env_var: str, default: float) -> float: + """Float counterpart of `get_env_int`.""" + value: Final = get_env_float_or_none(env_var) + return default if value is None else value def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int) -> int: @@ -41,19 +83,3 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int) default, ) return default - - -def get_env_int_or_none(env_var: str) -> int | None: - """Parse an environment variable as an integer, returning None when it is unset or unusable. - - Use this instead of `get_env_int` when callers must distinguish "explicitly configured" - from "left at the default", for example when an override should take precedence over a - value resolved from somewhere else. - """ - raw: Final = os.getenv(env_var) - if raw is None: - return None - try: - return int(raw.strip()) - except (ValueError, TypeError): - return None diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index 7545a1117a1..89794fc2480 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -21,6 +21,7 @@ from typing_extensions import ReadOnly, Required, assert_never import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS, AGENT_HEALTH_CHECK_TIMEOUT_SECONDS from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._types import ( CommonProxyErrors, @@ -228,10 +229,6 @@ def _check_agent_management_permission(user_api_key_dict: UserAPIKeyAuth) -> Non ) -AGENT_HEALTH_CHECK_TIMEOUT_SECONDS: Final = float(os.environ.get("LITELLM_AGENT_HEALTH_CHECK_TIMEOUT", "5.0")) -AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS = float(os.environ.get("LITELLM_AGENT_HEALTH_CHECK_GATHER_TIMEOUT", "30.0")) - - class _AgentHealthResult(TypedDict, total=False): agent_id: Required[str] healthy: Required[bool] diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cf90d90ea33..20504982fde 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -55,6 +55,7 @@ from litellm import _custom_logger_compatible_callbacks_literal from litellm.constants import ( DEFAULT_MODEL_CREATED_AT_TIME, FILE_USAGE_MAX_TRACKED_COUNTERS, + LITELLM_CONFIG_CACHE_TTL_SECONDS, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, MAX_TEAM_LIST_LIMIT, PROXY_REJECTED_BEFORE_ROUTING_KEY, @@ -150,6 +151,7 @@ from litellm.litellm_core_utils.core_helpers import ( is_expected_client_error, ) from litellm.litellm_core_utils.duration_parser import get_budget_window_start +from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -4505,7 +4507,6 @@ async def _lookup_deprecated_key( # DualCache for LiteLLM_Config param_name reads. # Redis layer is attached in proxy_server._init_cache. -LITELLM_CONFIG_CACHE_TTL_SECONDS: Final[int] = int(os.environ.get("LITELLM_CONFIG_PARAM_CACHE_TTL_SECONDS", "60")) _CONFIG_CACHE_MISS: Final[str] = "__litellm_config_param_miss__" litellm_config_cache: Final[DualCache] = DualCache( @@ -4789,28 +4790,28 @@ class PrismaClient: self._db_health_watchdog_task: asyncio.Task | None = None self._view_setup_task: asyncio.Task[_ViewSetupOutcome] | None = None self._db_last_reconnect_attempt_ts: float = 0.0 - self._db_reconnect_cooldown_seconds: int = max(1, int(os.getenv("PRISMA_RECONNECT_COOLDOWN_SECONDS", "15"))) + self._db_reconnect_cooldown_seconds: int = max(1, get_env_int("PRISMA_RECONNECT_COOLDOWN_SECONDS", 15)) self._db_read_only_recreate_ts: float = 0.0 self._db_read_only_recreate_streak: int = 0 self._db_health_watchdog_interval_seconds: int = max( - 5, int(os.getenv("PRISMA_HEALTH_WATCHDOG_INTERVAL_SECONDS", "30")) + 5, get_env_int("PRISMA_HEALTH_WATCHDOG_INTERVAL_SECONDS", 30) ) self._db_health_watchdog_enabled: bool = ( str_to_bool(os.getenv("PRISMA_HEALTH_WATCHDOG_ENABLED", "true")) is True ) self._db_health_watchdog_probe_timeout_seconds: float = max( 0.5, - float(os.getenv("PRISMA_HEALTH_WATCHDOG_PROBE_TIMEOUT_SECONDS", "5.0")), + get_env_float("PRISMA_HEALTH_WATCHDOG_PROBE_TIMEOUT_SECONDS", 5.0), ) self._db_watchdog_reconnect_timeout_seconds: float = max( - 1.0, float(os.getenv("PRISMA_WATCHDOG_RECONNECT_TIMEOUT_SECONDS", "30.0")) + 1.0, get_env_float("PRISMA_WATCHDOG_RECONNECT_TIMEOUT_SECONDS", 30.0) ) self._db_auth_reconnect_timeout_seconds: float = max( - 0.5, float(os.getenv("PRISMA_AUTH_RECONNECT_TIMEOUT_SECONDS", "2.0")) + 0.5, get_env_float("PRISMA_AUTH_RECONNECT_TIMEOUT_SECONDS", 2.0) ) self._db_auth_reconnect_lock_timeout_seconds: float = max( 0.0, - float(os.getenv("PRISMA_AUTH_RECONNECT_LOCK_TIMEOUT_SECONDS", "0.1")), + get_env_float("PRISMA_AUTH_RECONNECT_LOCK_TIMEOUT_SECONDS", 0.1), ) self._consecutive_reconnect_failures: int = 0 # Last generation of each read engine whose repair was attempted and @@ -4821,7 +4822,7 @@ class PrismaClient: # back to a caller whose engine is still unrepaired. Bounded at two # entries: a client has one writer and at most one reader. self._failed_recreate_generations: Mapping[PrismaWrapper, int] = MappingProxyType({}) - self._reconnect_escalation_threshold: int = max(1, int(os.getenv("PRISMA_RECONNECT_ESCALATION_THRESHOLD", "3"))) + self._reconnect_escalation_threshold: int = max(1, get_env_int("PRISMA_RECONNECT_ESCALATION_THRESHOLD", 3)) self._engine_pidfd: int = -1 self._engine_pid: int = 0 self._watching_engine: bool = False diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index e65a0dabd6d..6c86b058f0f 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -5,6 +5,7 @@ from types import MappingProxyType from typing import Final, Literal, cast from pydantic import ConfigDict, Field, field_validator, model_validator +from pydantic.fields import FieldInfo from typing_extensions import ReadOnly, Required, TypedDict from litellm._logging import verbose_logger @@ -1000,6 +1001,16 @@ def runtime_stream_scope( LoggingOnlyScope = Literal["input", "output"] +def _is_blank(value: object) -> bool: + return isinstance(value, str) and not value.strip() + + +def _default_if_blank(field: FieldInfo | None, value: object) -> object: + if field is None or field.is_required() or not _is_blank(value): + return value + return field.get_default(call_default_factory=True) + + class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch update guardrails api_key: str | None = Field(default=None, description="API key for the guardrail service") api_base: str | None = Field(default=None, description="Base URL for the guardrail service API") @@ -1288,6 +1299,14 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up ), ) + @model_validator(mode="before") # pyright: ignore[reportUntypedFunctionDecorator] # pydantic decorators are untyped + @classmethod + def blank_values_mean_field_default(cls, data: object) -> object: + if not isinstance(data, Mapping): + return data + fields: Final = cast(Mapping[str, object], data) # cast-ok: pydantic hands a before-validator the raw mapping + return {key: _default_if_blank(cls.model_fields.get(key), value) for key, value in fields.items()} + @field_validator( "mode", "default_action", @@ -1398,14 +1417,14 @@ class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # o ) return self - def __init__(self, **kwargs) -> None: + def __init__(self, **kwargs: object) -> None: default_on: Final = kwargs.pop("default_on", None) - if default_on is not None: + if default_on is not None and not _is_blank(default_on): kwargs["default_on"] = default_on else: kwargs["default_on"] = False - super().__init__(**kwargs) + super().__init__(**kwargs) # pyright: ignore[reportArgumentType] # pydantic validates each field's type at runtime def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/tests/unit/litellm_core_utils/test_env_utils.py b/tests/unit/litellm_core_utils/test_env_utils.py new file mode 100644 index 00000000000..1ddd172e274 --- /dev/null +++ b/tests/unit/litellm_core_utils/test_env_utils.py @@ -0,0 +1,96 @@ +import logging +from collections.abc import Callable, Iterator + +import pytest + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.env_utils import ( + get_env_float, + get_env_float_or_none, + get_env_int, + get_env_int_or_none, +) + + +@pytest.fixture +def litellm_log(caplog: pytest.LogCaptureFixture) -> Iterator[pytest.LogCaptureFixture]: + verbose_logger.addHandler(caplog.handler) + try: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + yield caplog + finally: + verbose_logger.removeHandler(caplog.handler) + + +def _logged_messages(caplog: pytest.LogCaptureFixture) -> list[str]: + return [record.getMessage() for record in dict.fromkeys(caplog.records)] + + +@pytest.mark.parametrize("raw", ["", " ", "\t"]) +def test_blank_numeric_env_value_means_unset_without_warning( + monkeypatch: pytest.MonkeyPatch, litellm_log: pytest.LogCaptureFixture, raw: str +) -> None: + monkeypatch.setenv("ENV_UTILS_TEST_KNOB", raw) + + assert get_env_float("ENV_UTILS_TEST_KNOB", 1.5) == 1.5 + assert get_env_int("ENV_UTILS_TEST_KNOB", 7) == 7 + assert get_env_float_or_none("ENV_UTILS_TEST_KNOB") is None + assert get_env_int_or_none("ENV_UTILS_TEST_KNOB") is None + assert _logged_messages(litellm_log) == [] + + +def test_set_numeric_env_value_is_parsed(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ENV_UTILS_TEST_FLOAT", " 2.5 ") + monkeypatch.setenv("ENV_UTILS_TEST_INT", " 12 ") + + assert get_env_float("ENV_UTILS_TEST_FLOAT", 1.5) == 2.5 + assert get_env_int("ENV_UTILS_TEST_INT", 7) == 12 + assert get_env_float_or_none("ENV_UTILS_TEST_FLOAT") == 2.5 + assert get_env_int_or_none("ENV_UTILS_TEST_INT") == 12 + + +@pytest.mark.parametrize( + ("read", "raw", "expected"), + [ + (lambda name: get_env_int(name, 7), "fast-int", 7), + (lambda name: get_env_float(name, 1.5), "fast-float", 1.5), + (get_env_int_or_none, "fast-int-or-none", None), + (get_env_float_or_none, "fast-float-or-none", None), + ], +) +def test_unparsable_numeric_env_value_falls_back_and_warns_once( + monkeypatch: pytest.MonkeyPatch, + litellm_log: pytest.LogCaptureFixture, + read: Callable[[str], float | None], + raw: str, + expected: float | None, +) -> None: + monkeypatch.setenv("ENV_UTILS_TEST_KNOB", raw) + + assert read("ENV_UTILS_TEST_KNOB") == expected + assert read("ENV_UTILS_TEST_KNOB") == expected + + warnings = [message for message in _logged_messages(litellm_log) if raw in message] + assert len(warnings) == 1 + assert "ENV_UTILS_TEST_KNOB" in warnings[0] + + +@pytest.mark.parametrize("env_var", ["MAX_STRING_LENGTH_STDOUT_LOG", "MAX_BASE64_LENGTH_STDOUT_LOG"]) +def test_unparsable_stdout_log_limit_does_not_break_logging( + monkeypatch: pytest.MonkeyPatch, litellm_log: pytest.LogCaptureFixture, env_var: str +) -> None: + """ + Regression: the LiteLLM loggers' own filter reads these limits through get_env_int on every + record, so warning about a garbage value re-entered the filter and every log call raised + RecursionError. + """ + raw = f"not-a-number-{env_var}" + monkeypatch.setenv(env_var, raw) + + verbose_logger.warning("first line") + verbose_logger.warning("second line") + + messages = _logged_messages(litellm_log) + assert "first line" in messages + assert "second line" in messages + assert len([message for message in messages if raw in message]) == 1 diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_straiker.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_straiker.py index 292dd54105f..85886a2b9af 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_straiker.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_straiker.py @@ -1,5 +1,6 @@ import json from types import MappingProxyType, SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -1216,7 +1217,25 @@ def test_v3_initializer_reads_api_version_from_config(): assert g._webhook_url().endswith("/api/v3/detect") -@pytest.mark.parametrize("api_version", ["2024-09-01", "", "v2"]) +@pytest.mark.parametrize(("api_key", "expected"), [("c4ac433a-uuid", "v1"), (V3_KEY, "v3")]) +def test_blank_api_version_follows_key_prefix_without_warning( + api_key: str, expected: str, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm + from litellm._logging import verbose_proxy_logger + from litellm.types.guardrails import Guardrail, LitellmParams + + monkeypatch.setattr(litellm, "callbacks", litellm.callbacks.copy()) + params: Final = LitellmParams(guardrail="straiker", mode="pre_call", api_key=api_key, api_version="") + + with patch.object(verbose_proxy_logger, "warning") as warning: + g = initialize_guardrail(params, Guardrail(guardrail_name="straiker", litellm_params=params)) + + assert g.api_version == expected + warning.assert_not_called() + + +@pytest.mark.parametrize("api_version", ["2024-09-01", "v2"]) @pytest.mark.parametrize(("api_key", "expected"), [("c4ac433a-uuid", "v1"), (V3_KEY, "v3")]) def test_unknown_api_version_follows_key_prefix(api_version, api_key, expected, monkeypatch): import litellm diff --git a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py index 13049093d6c..4401e3e95c0 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py @@ -1670,6 +1670,31 @@ async def test_patch_guardrail_clears_scope_when_logging_only_mode_is_removed( handler.delete_in_memory_guardrail(stored_guardrail["guardrail_id"]) +@pytest.mark.asyncio +async def test_patch_guardrail_with_blank_value_clears_the_stored_value(mocker, monkeypatch, mock_guardrail_registry): + """ + Regression: blank litellm_params were dropped before validation, so a PATCH that cleared + api_base with "" returned 200 but kept the stored value. + """ + handler, stored_guardrail = _setup_patch_scope_guardrail( + mocker, + monkeypatch, + mock_guardrail_registry, + _PatchScopeSupportedGuardrail, + "patch_blank_value_test", + {"mode": "pre_call", "api_base": "https://old.example.com"}, + ) + request = PatchGuardrailRequest.model_validate({"litellm_params": {"api_base": ""}}) + + try: + await patch_guardrail(stored_guardrail["guardrail_id"], request, user_api_key_dict=MOCK_ADMIN_USER) + + persisted_guardrail = mock_guardrail_registry.update_guardrail_in_db.call_args.kwargs["guardrail"] + assert persisted_guardrail["litellm_params"].api_base is None + finally: + handler.delete_in_memory_guardrail(stored_guardrail["guardrail_id"]) + + @pytest.mark.asyncio async def test_patch_guardrail_tolerates_stored_unsupported_scope_on_unrelated_update( mocker, monkeypatch, mock_guardrail_registry, caplog diff --git a/tests/unit/proxy/guardrails/test_init_guardrails.py b/tests/unit/proxy/guardrails/test_init_guardrails.py index c8eeaf11fde..daf7ef1fd05 100644 --- a/tests/unit/proxy/guardrails/test_init_guardrails.py +++ b/tests/unit/proxy/guardrails/test_init_guardrails.py @@ -371,6 +371,44 @@ def test_init_guardrails_v2_skips_invalid_guardrail_instead_of_crashing_boot(): assert "healthy_presidio" in guardrail_names +def test_init_guardrails_v2_keeps_guardrail_whose_env_values_are_blank(): + """ + Regression: a config.yaml guardrail with `default_on: os.environ/GUARDRAIL_DEFAULT_ON` and + that variable exported blank (a Helm `value: ""`) resolved to "", pydantic rejected it on the + bool field, and the proxy logged the guardrail and booted without it. Blank now means unset, + so the guardrail boots with its defaults. + """ + from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER + + IN_MEMORY_GUARDRAIL_HANDLER.IN_MEMORY_GUARDRAILS.clear() + IN_MEMORY_GUARDRAIL_HANDLER.guardrail_id_to_custom_guardrail.clear() + + all_guardrails = [ + { + "guardrail_name": "presidio_with_blank_env_values", + "litellm_params": { + "guardrail": SupportedGuardrailIntegrations.PRESIDIO.value, + "mode": "pre_call", + "default_on": "", + "mask_request_content": "", + "presidio_analyzer_api_base": "https://fakelink.com/v1/presidio/analyze", + "presidio_anonymizer_api_base": "https://fakelink.com/v1/presidio/anonymize", + }, + }, + ] + + init_guardrails_v2(all_guardrails=all_guardrails) + + initialized = { + guardrail["guardrail_name"]: guardrail + for guardrail in IN_MEMORY_GUARDRAIL_HANDLER.IN_MEMORY_GUARDRAILS.values() + } + assert "presidio_with_blank_env_values" in initialized + litellm_params = initialized["presidio_with_blank_env_values"]["litellm_params"] + assert litellm_params.default_on is False + assert litellm_params.mask_request_content is None + + def test_init_guardrails_v2_stops_boot_when_a_custom_code_guardrail_does_not_compile(): from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER diff --git a/tests/unit/test_constants.py b/tests/unit/test_constants.py index 18ed6975605..0d0e77765b7 100644 --- a/tests/unit/test_constants.py +++ b/tests/unit/test_constants.py @@ -17,6 +17,19 @@ from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outco from unittest.mock import patch +_ENV_HELPERS = frozenset( + {"get_env_int", "get_env_float", "get_env_int_in_range", "get_env_int_or_none", "get_env_float_or_none"} +) + + +def _single_name_target(node: ast.stmt) -> str | None: + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + return node.target.id + if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + return node.targets[0].id + return None + + def _build_constant_env_var_map() -> dict[str, str]: """ Build a mapping of CONSTANT_NAME -> ENV_VAR_NAME by parsing constants.py. @@ -29,13 +42,10 @@ def _build_constant_env_var_map() -> dict[str, str]: parsed = ast.parse(constants_source) for node in parsed.body: - if not isinstance(node, ast.Assign): + constant_name = _single_name_target(node) + if constant_name is None or not isinstance(node, (ast.Assign, ast.AnnAssign)) or node.value is None: continue - if len(node.targets) != 1 or not isinstance(node.targets[0], ast.Name): - continue - - constant_name = node.targets[0].id env_var_name = None for child in ast.walk(node.value): @@ -55,10 +65,10 @@ def _build_constant_env_var_map() -> dict[str, str]: env_var_name = child.args[0].value break - # get_env_int("ENV_NAME", default) + # get_env_int("ENV_NAME", default) and its siblings if ( isinstance(child.func, ast.Name) - and child.func.id == "get_env_int" + and child.func.id in _ENV_HELPERS and len(child.args) >= 1 and isinstance(child.args[0], ast.Constant) and isinstance(child.args[0].value, str) @@ -79,8 +89,10 @@ def _build_constant_env_var_map() -> dict[str, str]: (None, "48", 48), (None, None, 24), ("48", "72", 48), + (" ", None, 24), + ("", "48", 48), ], - ids=("canonical-only", "alias-only", "default", "canonical-wins"), + ids=("canonical-only", "alias-only", "default", "canonical-wins", "blank-canonical", "blank-canonical-alias-set"), ) def test_cli_jwt_expiration_hours_from_environment( monkeypatch: pytest.MonkeyPatch, @@ -105,12 +117,79 @@ def test_cli_jwt_expiration_hours_from_environment( importlib.reload(litellm.constants) +def _is_blank(value: object) -> bool: + return isinstance(value, str) and not value.strip() + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_constants_import_treats_blank_env_values_as_unset(blank: str) -> None: + """ + Regression: a numeric knob exported as a blank string (a Helm `value: ""` or an emptied .env + line) raised ValueError while importing litellm.constants, so the proxy crash-looped at boot. + Blank now means unset, so import succeeds with every env-backed constant blank, and every + numeric or optional constant keeps the value it has with the variable absent. + """ + constant_env_vars = _build_constant_env_var_map() + assert {"ROUTER_MAX_FALLBACKS", "LITELLM_MAX_STREAMING_DURATION_SECONDS"} <= constant_env_vars.keys() + env_vars = set(constant_env_vars.values()) + try: + with mock.patch.dict(os.environ): + for env_var in env_vars: + os.environ.pop(env_var, None) + importlib.reload(litellm.constants) + unset = {name: getattr(litellm.constants, name) for name in constant_env_vars} + with mock.patch.dict(os.environ, {env_var: blank for env_var in env_vars}): + importlib.reload(litellm.constants) + with_blank_env = {name: getattr(litellm.constants, name) for name in constant_env_vars} + finally: + importlib.reload(litellm.constants) + + numeric_or_optional = {name for name, value in unset.items() if value is None or type(value) in (int, float)} + changed = { + name: (unset[name], with_blank_env[name]) + for name in numeric_or_optional + if with_blank_env[name] != unset[name] and not _is_blank(with_blank_env[name]) + } + assert changed == {} + + +@pytest.mark.parametrize( + ("raw", "expected_timeout", "expected_explicit"), + [ + (None, constants.DEFAULT_REQUEST_TIMEOUT_SECONDS, False), + ("", constants.DEFAULT_REQUEST_TIMEOUT_SECONDS, False), + (" ", constants.DEFAULT_REQUEST_TIMEOUT_SECONDS, False), + ("fast", constants.DEFAULT_REQUEST_TIMEOUT_SECONDS, False), + (" 30 ", 30.0, True), + ], +) +def test_request_timeout_counts_as_explicit_only_when_it_parses( + monkeypatch: pytest.MonkeyPatch, raw: str | None, expected_timeout: float, expected_explicit: bool +) -> None: + """ + Regression: a blank REQUEST_TIMEOUT fell back to the 6000s sentinel but was still flagged as + explicitly set, so chat completion() kept the 6000s sentinel instead of its 600s fallback. + """ + if raw is None: + monkeypatch.delenv("REQUEST_TIMEOUT", raising=False) + else: + monkeypatch.setenv("REQUEST_TIMEOUT", raw) + try: + importlib.reload(litellm.constants) + assert litellm.constants.request_timeout == expected_timeout + assert litellm.constants.request_timeout_explicitly_set is expected_explicit + finally: + monkeypatch.undo() + importlib.reload(litellm.constants) + + @pytest.fixture() def _vcr_outcome_gate(request, vcr): install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -120,6 +199,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -153,6 +233,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -176,17 +257,12 @@ def _populate_known_models_for_morph_tests(monkeypatch: pytest.MonkeyPatch) -> N monkeypatch.setattr( litellm, "models_by_provider", - { - provider: models.copy() - for provider, models in litellm.models_by_provider.items() - }, + {provider: models.copy() for provider, models in litellm.models_by_provider.items()}, ) litellm.add_known_models() -@pytest.mark.usefixtures( - "_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests" -) +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests") def test_morph_config_get_provider_info(): """Test that MorphChatConfig returns correct provider info.""" config = MorphChatConfig() @@ -207,9 +283,8 @@ def test_morph_config_get_provider_info(): assert api_base == "https://custom.morph.com" assert api_key == "key" -@pytest.mark.usefixtures( - "_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests" -) + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests") def test_morph_get_llm_provider(): """Test that get_llm_provider correctly identifies morph models.""" # Test with morph/model format @@ -219,9 +294,8 @@ def test_morph_get_llm_provider(): _, custom_llm_provider, _, _ = get_llm_provider("morph/morph-v3-fast") assert custom_llm_provider == "morph" -@pytest.mark.usefixtures( - "_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests" -) + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests") def test_morph_in_provider_lists(): """Test that morph is included in all necessary provider lists.""" import litellm @@ -242,9 +316,8 @@ def test_morph_in_provider_lists(): # Check models are in model_list after initialization assert all(model in litellm.model_list for model in ["morph/morph-v3-large", "morph/morph-v3-fast"]) -@pytest.mark.usefixtures( - "_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests" -) + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests") def test_morph_supported_params(): """Test that MorphChatConfig returns correct supported parameters.""" config = MorphChatConfig() @@ -258,9 +331,8 @@ def test_morph_supported_params(): assert all(param in supported_params for param in expected_params) -@pytest.mark.usefixtures( - "_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests" -) + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests") def test_morph_custom_llm_provider(): """Test that morph models are correctly identified.""" config = MorphChatConfig() diff --git a/tests/unit/types/test_guardrails.py b/tests/unit/types/test_guardrails.py index 8b74f7ce0fd..5c652a10e16 100644 --- a/tests/unit/types/test_guardrails.py +++ b/tests/unit/types/test_guardrails.py @@ -13,6 +13,7 @@ import litellm.litellm_core_utils import litellm.litellm_core_utils.litellm_logging from litellm import completion from litellm.integrations.custom_logger import CustomLogger +from litellm.types.guardrails import BaseLitellmParams, LitellmParams from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -157,3 +158,23 @@ def setup_and_teardown(): if hasattr(litellm, "in_memory_llm_clients_cache"): litellm.in_memory_llm_clients_cache.flush_cache() yield + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_litellm_params_treats_blank_values_as_unset(blank: str) -> None: + """ + Regression: `os.environ/X` with X exported blank resolves to "", which pydantic rejected on + bool fields (default_on, mask_request_content, fail_on_error), so the guardrail was dropped at + boot. Blank now means unset, so the field keeps the same default as when it is omitted. + """ + with_blank_values = LitellmParams( + guardrail="presidio", + mode="pre_call", + default_on=blank, + mask_request_content=blank, + api_key=blank, + ) + omitted = LitellmParams(guardrail="presidio", mode="pre_call") + assert with_blank_values.model_dump() == omitted.model_dump() + + assert BaseLitellmParams(fail_on_error=blank).model_dump() == BaseLitellmParams().model_dump()