From 493ed616bbf306f7c0f98f0577985b5bec6d3ac8 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 14 Oct 2026 12:49:31 -0700 Subject: [PATCH] fix(config): treat blank env values as unset so the proxy neither crash-loops nor silently drops a guardrail (#46341) * fix(config): treat blank numeric env values as unset instead of crashing at import A numeric knob exported as an empty string, which is what a Helm value: "" or an emptied .env line produces, raised ValueError inside litellm/constants.py and the proxy crash-looped before serving traffic. Every int(os.getenv(...)) and float(os.getenv(...)) in constants.py now goes through get_env_int / get_env_float, which treat blank as unset and fall back to the default with a warning on garbage. * fix(guardrails): treat blank litellm_params values as unset so the guardrail still loads A guardrail whose litellm_params read os.environ/X resolves X to "" when the variable is exported but empty. Pydantic rejected "" for bool and typed fields such as default_on and fail_on_error, the boot log reported the guardrail as skipped and the proxy came up without it. A before-validator on BaseLitellmParams now drops blank string values so each field takes its default, and the default_on shortcut in LitellmParams treats a blank value as False. * fix(config): warn once about an unparsable numeric env value so logging cannot recurse The LiteLLM loggers' filter reads MAX_STRING_LENGTH_STDOUT_LOG and MAX_BASE64_LENGTH_STDOUT_LOG through get_env_int on every record. Warning about a garbage value from inside that read re-entered the filter, so every log call raised RecursionError. The value is now marked as reported before the warning is logged, which also stops per-record readers from repeating it. get_env_int_or_none and the new get_env_float_or_none share the same parser * fix(config): treat blank values as unset for the remaining import-time numeric knobs LITELLM_MAX_STREAMING_DURATION_SECONDS, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES, a whitespace CLI_JWT_EXPIRATION_HOURS, LITELLM_CONFIG_PARAM_CACHE_TTL_SECONDS, the agent health check timeouts and the Prisma reconnect knobs still crashed the proxy at boot when blank. A blank or unparsable REQUEST_TIMEOUT no longer counts as explicitly set, so chat completion() keeps its 600s fallback * fix(guardrails): map blank litellm_params to the field default so PATCH can still clear a value Dropping blank keys before validation made PATCH /guardrails/{id} with api_base "" a silent no-op. Blank values of optional fields now resolve to the field default and stay in the set fields, so the PATCH clears the stored value. Required fields and unknown keys keep the value as given * test: cover proxy numeric knobs in-process and expect no warning for a blank Straiker api_version The three proxy import-time numeric knobs move into litellm/constants.py so the existing in-process blank-env test covers them, replacing the subprocess test. A blank api_version now means unset, so the Straiker test asserts the key-prefix version with no warning. * fix(guardrails): keep LitellmParams kwargs typed without adding argument-type errors --------- Co-authored-by: yuneng-jiang --- litellm/constants.py | 616 +++++++++--------- litellm/litellm_core_utils/env_utils.py | 82 ++- litellm/proxy/agent_endpoints/endpoints.py | 5 +- litellm/proxy/utils.py | 17 +- litellm/types/guardrails.py | 25 +- .../unit/litellm_core_utils/test_env_utils.py | 96 +++ .../guardrail_hooks/test_straiker.py | 21 +- .../guardrails/test_guardrail_endpoints.py | 25 + .../proxy/guardrails/test_init_guardrails.py | 38 ++ tests/unit/test_constants.py | 126 +++- tests/unit/types/test_guardrails.py | 21 + 11 files changed, 679 insertions(+), 393 deletions(-) create mode 100644 tests/unit/litellm_core_utils/test_env_utils.py diff --git a/litellm/constants.py b/litellm/constants.py index 9d18fe5489d..4ac6adaba73 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -4,7 +4,13 @@ from enum import Enum from types import MappingProxyType from typing import Final, Literal -from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_in_range, get_env_int_or_none +from litellm.litellm_core_utils.env_utils import ( + get_env_float, + get_env_float_or_none, + get_env_int, + get_env_int_in_range, + get_env_int_or_none, +) MICROSOFT_GRAPH_BETA_BASE: Final = "https://graph.microsoft.com/beta" OAUTH_TOKEN_EXCHANGE_CACHE_SAFETY_MARGIN_SECONDS: Final = 60 @@ -32,7 +38,7 @@ DEFER_PYDANTIC_BUILD: Final = os.getenv("DEFER_PYDANTIC_BUILD", "true") in ("tru DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"}) -ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5)) +ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5) ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000 RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset( { @@ -68,10 +74,10 @@ ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG: Final[frozenset[str]] = frozenset( "auto_router_capability_limit", } ) -DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) -DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) +DEFAULT_BATCH_SIZE: Final = get_env_int("DEFAULT_BATCH_SIZE", 512) +DEFAULT_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_FLUSH_INTERVAL_SECONDS", 5) CLICKHOUSE_BATCH_SIZE: Final = get_env_int("CLICKHOUSE_BATCH_SIZE", 10_000) -CLICKHOUSE_FLUSH_INTERVAL_SECONDS: Final = float(os.getenv("CLICKHOUSE_FLUSH_INTERVAL_SECONDS", "1.0")) +CLICKHOUSE_FLUSH_INTERVAL_SECONDS: Final = get_env_float("CLICKHOUSE_FLUSH_INTERVAL_SECONDS", 1.0) CLICKHOUSE_MAX_BUFFERED_ROWS: Final = get_env_int("CLICKHOUSE_MAX_BUFFERED_ROWS", 200_000) CLICKHOUSE_MAX_RETRIES: Final = get_env_int("CLICKHOUSE_MAX_RETRIES", 3) DEFAULT_CLICKHOUSE_DATABASE: Final = "litellm" @@ -89,9 +95,9 @@ LENS_DATASET_MAX_CASE_CHARS: Final = get_env_int("LENS_DATASET_MAX_CASE_CHARS", LENS_DATASET_TRACE_PAGE_SIZE: Final = 500 LENS_FEEDBACK_MAX_COMMENT_CHARS: Final = 10_000 LENS_FEEDBACK_MAX_SCORE: Final = 10 -DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) -DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) -DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = int(os.getenv("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", "16")) +DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10) +DEFAULT_S3_BATCH_SIZE: Final = get_env_int("DEFAULT_S3_BATCH_SIZE", 512) +DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = get_env_int("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", 16) DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY: Final = get_env_int("DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY", 200) S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS: Final = get_env_int_in_range("S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS", 30, 1, 3600) # https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html @@ -103,48 +109,43 @@ MAX_S3_OBJECT_DOWNLOAD_FILENAME_BYTES: Final = 1024 S3_LOG_PROMPTS_ONLY_ENV_VAR: Final = "S3_LOG_PROMPTS_ONLY" S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY" MAX_FILE_LIST_LIMIT: Final = 10000 -DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10)) -DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1)) +DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = get_env_int("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10) +DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = get_env_int("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1) GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10 budget_reservation_disabled_info_emitted = False -DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1)) -DEFAULT_SQS_BATCH_SIZE: Final = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512)) +DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = get_env_int("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1) +DEFAULT_SQS_BATCH_SIZE: Final = get_env_int("DEFAULT_SQS_BATCH_SIZE", 512) SQS_SEND_MESSAGE_ACTION: Final = "SendMessage" SQS_API_VERSION: Final = "2012-11-05" -DEFAULT_MAX_RETRIES: Final = int(os.getenv("DEFAULT_MAX_RETRIES", 2)) +DEFAULT_MAX_RETRIES: Final = get_env_int("DEFAULT_MAX_RETRIES", 2) # Max records accepted in one POST /v1/callbacks/logs batch. Bounds the blast # radius: each record fans out to spend logs + every callback integration. MAX_CALLBACK_LOG_RECORDS: Final = 1000 -DEFAULT_MAX_RECURSE_DEPTH: Final = int(os.getenv("DEFAULT_MAX_RECURSE_DEPTH", 100)) +DEFAULT_MAX_RECURSE_DEPTH: Final = get_env_int("DEFAULT_MAX_RECURSE_DEPTH", 100) GUARDRAIL_ROTATION_ATTEMPTS: Final = 3 -DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER = int(os.getenv("DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER", 10)) -DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = float( - os.getenv("DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5) +DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER = get_env_int("DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER", 10) +DEFAULT_FAILURE_THRESHOLD_PERCENT: Final = get_env_float( + "DEFAULT_FAILURE_THRESHOLD_PERCENT", 0.5 ) # default cooldown a deployment if 50% of requests fail in a given minute -DEFAULT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_MAX_TOKENS", 4096)) -DEFAULT_ALLOWED_FAILS: Final = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3)) -DEFAULT_REDIS_SYNC_INTERVAL: Final = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1)) -DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5)) -DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = float( - os.getenv("DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS", "1") -) -DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5)) -DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)) -DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250)) +DEFAULT_MAX_TOKENS: Final = get_env_int("DEFAULT_MAX_TOKENS", 4096) +DEFAULT_ALLOWED_FAILS: Final = get_env_int("DEFAULT_ALLOWED_FAILS", 3) +DEFAULT_REDIS_SYNC_INTERVAL: Final = get_env_int("DEFAULT_REDIS_SYNC_INTERVAL", 1) +DEFAULT_COOLDOWN_TIME_SECONDS: Final = get_env_int("DEFAULT_COOLDOWN_TIME_SECONDS", 5) +DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = get_env_float("DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS", 1.0) +DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLING_RETRIES", 5) +DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1) +DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250) HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0 # Maximum wall-clock seconds a streaming response is allowed to run. # Streams exceeding this duration are terminated with a Timeout error. # None (default) = no limit. Set env var to a number of seconds to enable globally. -_max_stream_duration_env: Final = os.getenv("LITELLM_MAX_STREAMING_DURATION_SECONDS", None) -LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = ( - float(_max_stream_duration_env) if _max_stream_duration_env is not None else None -) +LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = get_env_float_or_none("LITELLM_MAX_STREAMING_DURATION_SECONDS") # Maximum number of base64 characters to keep in logging payloads. # Data URIs exceeding this are replaced with a size placeholder. # Set to 0 to disable truncation. -MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64)) +MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64) BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024 REDACTED_BY_LITELLM: Final = "redacted-by-litellm" # in-memory stand-in handed to provider converters for redacted arguments; never stored @@ -158,44 +159,44 @@ MAX_BASE64_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_BASE64_LENGTH_STDOUT_LOG" LITELLM_DETAILED_TIMING: Final = os.getenv("LITELLM_DETAILED_TIMING", "false").lower() == "true" # Model cost map validation constants -MODEL_COST_MAP_MIN_MODEL_COUNT: Final = int( - os.getenv("MODEL_COST_MAP_MIN_MODEL_COUNT", 50) +MODEL_COST_MAP_MIN_MODEL_COUNT: Final = get_env_int( + "MODEL_COST_MAP_MIN_MODEL_COUNT", 50 ) # Minimum number of models a fetched cost map must contain to be considered valid -MODEL_COST_MAP_MAX_SHRINK_RATIO: Final = float( - os.getenv("MODEL_COST_MAP_MAX_SHRINK_RATIO", 0.5) +MODEL_COST_MAP_MAX_SHRINK_RATIO: Final = get_env_float( + "MODEL_COST_MAP_MAX_SHRINK_RATIO", 0.5 ) # Maximum allowed shrinkage ratio vs local backup (0.5 = reject if fetched map is <50% of backup) -DEFAULT_IMAGE_WIDTH: Final = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300)) -DEFAULT_IMAGE_HEIGHT: Final = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300)) +DEFAULT_IMAGE_WIDTH: Final = get_env_int("DEFAULT_IMAGE_WIDTH", 300) +DEFAULT_IMAGE_HEIGHT: Final = get_env_int("DEFAULT_IMAGE_HEIGHT", 300) # Maximum size for image URL downloads in MB (default 50MB, set to 0 to disable limit) # This prevents memory issues from downloading very large images # Maps to OpenAI's 50 MB payload limit - requests with images exceeding this size will be rejected # Set MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0 to disable image URL handling entirely -MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: Final = float(os.getenv("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 50)) -MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB: Final = int( - os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024) +MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: Final = get_env_float("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 50.0) +MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB: Final = get_env_int( + "MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024 ) # 1MB = 1024KB # Surrogate-repair fallback in _read_request_body runs two full-body re.sub passes # that block the event loop on multi-MB malformed bodies. Skip the repair above this # size and raise the existing 400 immediately. Set to 0 to disable the cap. MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB: Final = get_env_int("MAX_REQUEST_BODY_SIZE_TO_REPAIR_MB", 1) -SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD: Final = int( - os.getenv("SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD", 1000) +SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD: Final = get_env_int( + "SINGLE_DEPLOYMENT_TRAFFIC_FAILURE_THRESHOLD", 1000 ) # Minimum number of requests to consider "reasonable traffic". Used for single-deployment cooldown logic. -DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS: Final = int( - os.getenv("DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS", 5) +DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS: Final = get_env_int( + "DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS", 5 ) # Minimum number of requests before applying error rate cooldown. Prevents cooldown from triggering on first failure. -DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET = int(os.getenv("DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET", 0)) +DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET = get_env_int("DEFAULT_REASONING_EFFORT_DISABLE_THINKING_BUDGET", 0) # MCP Semantic Tool Filter Defaults DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL: Final = str( os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_EMBEDDING_MODEL", "text-embedding-3-small") ) -DEFAULT_MCP_SEMANTIC_FILTER_TOP_K: Final = int(os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_TOP_K", 10)) -DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = float( - os.getenv("DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3) +DEFAULT_MCP_SEMANTIC_FILTER_TOP_K: Final = get_env_int("DEFAULT_MCP_SEMANTIC_FILTER_TOP_K", 10) +DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD: Final = get_env_float( + "DEFAULT_MCP_SEMANTIC_FILTER_SIMILARITY_THRESHOLD", 0.3 ) -MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH: Final = int(os.getenv("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150)) +MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH: Final = get_env_int("MAX_MCP_SEMANTIC_FILTER_TOOLS_HEADER_LENGTH", 150) MAX_LITELLM_CALL_ID_LENGTH: Final = 256 MAX_GUARDRAIL_SCAN_METADATA_HEADER_LENGTH: Final = 2048 @@ -205,15 +206,15 @@ DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS: Final = 2000 DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL: Final = str( os.getenv("DEFAULT_SEMANTIC_GUARD_EMBEDDING_MODEL", "text-embedding-3-small") ) -DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD", 0.75)) +DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD = get_env_float("DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD", 0.75) DEFAULT_OPENAI_MODERATIONS_MODEL: Final = "omni-moderation-latest" # MCP OAuth2 Client Credentials Defaults -MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS: Final = int(os.getenv("MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS", "60")) -MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE", "200")) -MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", "3600")) -MCP_SSO_ASSERTION_CACHE_TTL_SECONDS: Final = int(os.getenv("MCP_SSO_ASSERTION_CACHE_TTL_SECONDS", "60")) +MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS", 60) +MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE: Final = get_env_int("MCP_OAUTH2_TOKEN_CACHE_MAX_SIZE", 200) +MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL: Final = get_env_int("MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL", 3600) +MCP_SSO_ASSERTION_CACHE_TTL_SECONDS: Final = get_env_int("MCP_SSO_ASSERTION_CACHE_TTL_SECONDS", 60) # mcp_tool_permissions entry that grants every current and future tool on a server MCP_ALL_TOOLS_WILDCARD: Final = "*" @@ -222,20 +223,20 @@ MCP_ALL_TOOLS_WILDCARD: Final = "*" # npm/npx needs a writable cache dir; in containers the default (~/.npm) # may not exist or be read-only. /tmp is always writable. MCP_NPM_CACHE_DIR: Final = os.getenv("MCP_NPM_CACHE_DIR", "/tmp/.npm_mcp_cache") -MCP_OAUTH2_TOKEN_CACHE_MIN_TTL: Final = int(os.getenv("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", "10")) +MCP_OAUTH2_TOKEN_CACHE_MIN_TTL: Final = get_env_int("MCP_OAUTH2_TOKEN_CACHE_MIN_TTL", 10) # Per-user OAuth token Redis cache (for server-side token storage) MCP_PER_USER_TOKEN_REDIS_KEY_PREFIX: Final = "mcp:per_user_token" -MCP_PER_USER_TOKEN_DEFAULT_TTL: Final = int( - os.getenv("MCP_PER_USER_TOKEN_DEFAULT_TTL", "43200") # 12 hours -) -MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = int(os.getenv("MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS", "60")) +MCP_PER_USER_TOKEN_DEFAULT_TTL: Final = get_env_int("MCP_PER_USER_TOKEN_DEFAULT_TTL", 43200) +MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS", 60) # MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers. -MCP_CLIENT_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_CLIENT_TIMEOUT", "60.0")) -MCP_TOOL_LISTING_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_TOOL_LISTING_TIMEOUT", "30.0")) -MCP_METADATA_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_METADATA_TIMEOUT", "10.0")) -MCP_HEALTH_CHECK_TIMEOUT: Final = float(os.getenv("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", "10.0")) +MCP_CLIENT_TIMEOUT: Final = get_env_float("LITELLM_MCP_CLIENT_TIMEOUT", 60.0) +MCP_TOOL_LISTING_TIMEOUT: Final = get_env_float("LITELLM_MCP_TOOL_LISTING_TIMEOUT", 30.0) +MCP_METADATA_TIMEOUT: Final = get_env_float("LITELLM_MCP_METADATA_TIMEOUT", 10.0) +MCP_HEALTH_CHECK_TIMEOUT: Final = get_env_float("LITELLM_MCP_HEALTH_CHECK_TIMEOUT", 10.0) +AGENT_HEALTH_CHECK_TIMEOUT_SECONDS: Final = get_env_float("LITELLM_AGENT_HEALTH_CHECK_TIMEOUT", 5.0) +AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS: Final = get_env_float("LITELLM_AGENT_HEALTH_CHECK_GATHER_TIMEOUT", 30.0) MCP_UPSTREAM_SESSION_SHUTDOWN_TIMEOUT_SECONDS: Final = 5.0 MCP_TOOL_LISTING_MAX_PAGES: Final = 1000 MCP_GATEWAY_SESSION_ID_PREFIX_LENGTH: Final = 8 @@ -253,7 +254,7 @@ MCP_STDIO_ALLOWED_COMMANDS: Final[frozenset] = frozenset( ) # MCP OAuth2 Token Exchange (OBO) Defaults -MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE: Final = int(os.getenv("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", "500")) +MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE: Final = get_env_int("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", 500) LITELLM_UI_ALLOW_HEADERS: Final = [ "x-litellm-semantic-filter", @@ -266,14 +267,14 @@ LITELLM_UI_ALLOW_HEADERS: Final = [ ] # Gemini model-specific minimal thinking budget constants -DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH: Final = int( - os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH", 1) +DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH: Final = get_env_int( + "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH", 1 ) -DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO: Final = int( - os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128) +DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO: Final = get_env_int( + "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO", 128 ) -DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE: Final = int( - os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512) +DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE: Final = get_env_int( + "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE", 512 ) # Maximum number of callbacks that can be registered @@ -295,8 +296,8 @@ PROXY_LLM_PROVIDER_FALLBACK: Final = "litellm_proxy" PROXY_REJECTED_BEFORE_ROUTING_KEY: Final = "proxy_rejected_before_routing" # Generic fallback for unknown models -DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = int( - os.getenv("DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128) +DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET: Final = get_env_int( + "DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET", 128 ) # Provider-specific API base URLs @@ -315,19 +316,19 @@ OPEN_SANDBOX_DEFAULT_TIMEOUT: Final = 300 OPEN_SANDBOX_READY_TIMEOUT: Final = 30.0 OPEN_SANDBOX_POLL_INTERVAL: Final = 0.2 -DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = int(os.getenv("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024)) -DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET: Final = int( - os.getenv("DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET", 2048) +DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET = get_env_int("DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET", 1024) +DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET: Final = get_env_int( + "DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET", 2048 ) -DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = int(os.getenv("DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET", 4096)) -DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET = int(os.getenv("DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET", 8192)) -DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET = int(os.getenv("DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET", 16384)) -MAX_TOKEN_TRIMMING_ATTEMPTS: Final = int( - os.getenv("MAX_TOKEN_TRIMMING_ATTEMPTS", 10) +DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET = get_env_int("DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET", 4096) +DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET = get_env_int("DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET", 8192) +DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET = get_env_int("DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET", 16384) +MAX_TOKEN_TRIMMING_ATTEMPTS: Final = get_env_int( + "MAX_TOKEN_TRIMMING_ATTEMPTS", 10 ) # Maximum number of attempts to trim the message RUNWAYML_DEFAULT_API_VERSION: Final = str(os.getenv("RUNWAYML_DEFAULT_API_VERSION", "2024-11-06")) -RUNWAYML_POLLING_TIMEOUT = int(os.getenv("RUNWAYML_POLLING_TIMEOUT", 600)) # 10 minutes default for image generation +RUNWAYML_POLLING_TIMEOUT = get_env_int("RUNWAYML_POLLING_TIMEOUT", 600) # 10 minutes default for image generation ########## Networking constants ############################################################## _DEFAULT_TTL_FOR_HTTPX_CLIENTS: Final = 3600 # 1 hour, re-use the same httpx client for 1 hour @@ -344,10 +345,10 @@ EVICTED_LLM_CLIENT_CLOSE_MAX_PENDING: Final = 10_000 # Aiohttp connection pooling - prevents memory leaks from unbounded connection growth # Set to 0 for unlimited (not recommended for production) -AIOHTTP_CONNECTOR_LIMIT: Final = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 1000)) -AIOHTTP_CONNECTOR_LIMIT_PER_HOST: Final = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500)) -AIOHTTP_KEEPALIVE_TIMEOUT: Final = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 15)) -AIOHTTP_TTL_DNS_CACHE: Final = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300)) +AIOHTTP_CONNECTOR_LIMIT: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT", 1000) +AIOHTTP_CONNECTOR_LIMIT_PER_HOST: Final = get_env_int("AIOHTTP_CONNECTOR_LIMIT_PER_HOST", 500) +AIOHTTP_KEEPALIVE_TIMEOUT: Final = get_env_int("AIOHTTP_KEEPALIVE_TIMEOUT", 15) +AIOHTTP_TTL_DNS_CACHE: Final = get_env_int("AIOHTTP_TTL_DNS_CACHE", 300) # TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs # whose idle timeout is shorter than provider response timeouts (e.g. AWS NAT # Gateway: 350s vs OpenAI/Azure: 600s). Without this, the kernel sends nothing @@ -355,9 +356,9 @@ AIOHTTP_TTL_DNS_CACHE: Final = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300)) # arrives. Enabling SO_KEEPALIVE makes the kernel emit TCP probes that reset # the NAT idle timer. AIOHTTP_SO_KEEPALIVE: Final = os.getenv("AIOHTTP_SO_KEEPALIVE", "False").lower() == "true" -AIOHTTP_TCP_KEEPIDLE: Final = int(os.getenv("AIOHTTP_TCP_KEEPIDLE", 60)) -AIOHTTP_TCP_KEEPINTVL: Final = int(os.getenv("AIOHTTP_TCP_KEEPINTVL", 30)) -AIOHTTP_TCP_KEEPCNT: Final = int(os.getenv("AIOHTTP_TCP_KEEPCNT", 5)) +AIOHTTP_TCP_KEEPIDLE: Final = get_env_int("AIOHTTP_TCP_KEEPIDLE", 60) +AIOHTTP_TCP_KEEPINTVL: Final = get_env_int("AIOHTTP_TCP_KEEPINTVL", 30) +AIOHTTP_TCP_KEEPCNT: Final = get_env_int("AIOHTTP_TCP_KEEPCNT", 5) # enable_cleanup_closed is only needed for Python versions with the SSL leak bug # Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960) # Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78 @@ -370,10 +371,9 @@ AIOHTTP_NEEDS_CLEANUP_CLOSED: Final = (3, 13, 0) <= sys.version_info < ( # WebSocket constants # Default to None (unlimited) to match OpenAI's official agents SDK behavior # https://github.com/openai/openai-agents-python/blob/cf1b933660e44fd37b4350c41febab8221801409/src/agents/realtime/openai_realtime.py#L235 -_max_size_env: Final = os.getenv("REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES") -REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES: Final = int(_max_size_env) if _max_size_env is not None else None -REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = float( - os.getenv("REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", "20.0") +REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES: Final = get_env_int_or_none("REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES") +REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS: Final = get_env_float( + "REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS", 20.0 ) # RFC 6455 caps the close frame payload at 125 bytes, 2 of which carry the status code @@ -439,15 +439,16 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_sp REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer" REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer" REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer" -MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = int(os.getenv("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)) +MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = get_env_int("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100) REDIS_SPEND_LOGS_BUFFER_KEY: Final = "litellm_spend_logs_buffer" REDIS_SPEND_LOGS_BUFFER_MAX_ROWS: Final = 100000 REDIS_SPEND_LOGS_BUFFER_DEQUEUE_COUNT: Final = 1000 # Bounds asyncio.Queue() instances (log queues, spend update queues, etc.) to prevent unbounded memory growth -LITELLM_ASYNCIO_QUEUE_MAXSIZE: Final = int(os.getenv("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000)) -TOOL_POLICY_CACHE_TTL_SECONDS: Final = int(os.getenv("TOOL_POLICY_CACHE_TTL_SECONDS", 60)) -GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS: Final = int( - os.getenv("GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS", 24 * 60 * 60) +LITELLM_ASYNCIO_QUEUE_MAXSIZE: Final = get_env_int("LITELLM_ASYNCIO_QUEUE_MAXSIZE", 1000) +TOOL_POLICY_CACHE_TTL_SECONDS: Final = get_env_int("TOOL_POLICY_CACHE_TTL_SECONDS", 60) +LITELLM_CONFIG_CACHE_TTL_SECONDS: Final = get_env_int("LITELLM_CONFIG_PARAM_CACHE_TTL_SECONDS", 60) +GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS: Final = get_env_int( + "GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS", 24 * 60 * 60 ) BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS: Final = 25_000 CONTENT_FILTER_STREAMING_HOLDBACK_CHARS: Final = 50 @@ -457,8 +458,8 @@ PRESIDIO_ANALYZE_CHUNK_OVERLAP_CHARS: Final = 4096 PRESIDIO_ANALYZE_CHUNK_CONCURRENCY: Final = 8 # Aggregation threshold: default to 80% of the asyncio queue maxsize so the check can always trigger. # Must be < LITELLM_ASYNCIO_QUEUE_MAXSIZE; if set higher the aggregation logic will never fire. -MAX_SIZE_IN_MEMORY_QUEUE: Final = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8))) -MAX_IN_MEMORY_QUEUE_FLUSH_COUNT: Final = int(os.getenv("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000)) +MAX_SIZE_IN_MEMORY_QUEUE: Final = get_env_int("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8)) +MAX_IN_MEMORY_QUEUE_FLUSH_COUNT: Final = get_env_int("MAX_IN_MEMORY_QUEUE_FLUSH_COUNT", 1000) ############################################################################################### # Providers will not cache a prefix below a minimum size. That minimum is per-model, not global: # Anthropic's ranges from 512 to 4096 depending on the model, and can differ per platform for the @@ -473,22 +474,22 @@ MINIMUM_PROMPT_CACHE_TOKEN_COUNT: Final = ( else DEFAULT_MINIMUM_PROMPT_CACHE_TOKEN_COUNT ) PROMPT_CACHE_LOOKBACK_POSITIONS: Final = 20 -DEFAULT_TRIM_RATIO: Final = float( - os.getenv("DEFAULT_TRIM_RATIO", 0.75) +DEFAULT_TRIM_RATIO: Final = get_env_float( + "DEFAULT_TRIM_RATIO", 0.75 ) # default ratio of tokens to trim from the end of a prompt -HOURS_IN_A_DAY: Final = int(os.getenv("HOURS_IN_A_DAY", 24)) -DAYS_IN_A_WEEK: Final = int(os.getenv("DAYS_IN_A_WEEK", 7)) -DAYS_IN_A_MONTH: Final = int(os.getenv("DAYS_IN_A_MONTH", 28)) -DAYS_IN_A_YEAR: Final = int(os.getenv("DAYS_IN_A_YEAR", 365)) -REPLICATE_MODEL_NAME_WITH_ID_LENGTH: Final = int(os.getenv("REPLICATE_MODEL_NAME_WITH_ID_LENGTH", 64)) +HOURS_IN_A_DAY: Final = get_env_int("HOURS_IN_A_DAY", 24) +DAYS_IN_A_WEEK: Final = get_env_int("DAYS_IN_A_WEEK", 7) +DAYS_IN_A_MONTH: Final = get_env_int("DAYS_IN_A_MONTH", 28) +DAYS_IN_A_YEAR: Final = get_env_int("DAYS_IN_A_YEAR", 365) +REPLICATE_MODEL_NAME_WITH_ID_LENGTH: Final = get_env_int("REPLICATE_MODEL_NAME_WITH_ID_LENGTH", 64) #### TOKEN COUNTING #### -FUNCTION_DEFINITION_TOKEN_COUNT: Final = int(os.getenv("FUNCTION_DEFINITION_TOKEN_COUNT", 9)) -SYSTEM_MESSAGE_TOKEN_COUNT: Final = int(os.getenv("SYSTEM_MESSAGE_TOKEN_COUNT", 4)) -TOOL_CHOICE_OBJECT_TOKEN_COUNT: Final = int(os.getenv("TOOL_CHOICE_OBJECT_TOKEN_COUNT", 4)) -DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT", 10)) -DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", 20)) -MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES: Final = int(os.getenv("MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES", 768)) -MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES: Final = int(os.getenv("MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES", 2000)) +FUNCTION_DEFINITION_TOKEN_COUNT: Final = get_env_int("FUNCTION_DEFINITION_TOKEN_COUNT", 9) +SYSTEM_MESSAGE_TOKEN_COUNT: Final = get_env_int("SYSTEM_MESSAGE_TOKEN_COUNT", 4) +TOOL_CHOICE_OBJECT_TOKEN_COUNT: Final = get_env_int("TOOL_CHOICE_OBJECT_TOKEN_COUNT", 4) +DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT: Final = get_env_int("DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT", 10) +DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT: Final = get_env_int("DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT", 20) +MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES: Final = get_env_int("MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES", 768) +MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES: Final = get_env_int("MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES", 2000) # tiktoken's BPE merge loop is quadratic in the length of a single regex piece, so a long run of one # repeated character (dot leaders, whitespace, zero-padded base64) can take minutes on a multi-MB payload. # Encoding in chunks makes the cost linear, at a drift of at most ~1 token per chunk boundary. The upper @@ -512,69 +513,63 @@ TOKEN_COUNTER_MAX_CONCURRENT_COUNTS: Final = get_env_int_in_range( minimum=1, maximum=256, ) -MAX_TILE_WIDTH: Final = int(os.getenv("MAX_TILE_WIDTH", 512)) -MAX_TILE_HEIGHT: Final = int(os.getenv("MAX_TILE_HEIGHT", 512)) -OPENAI_FILE_SEARCH_COST_PER_1K_CALLS: Final = float(os.getenv("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000)) +MAX_TILE_WIDTH: Final = get_env_int("MAX_TILE_WIDTH", 512) +MAX_TILE_HEIGHT: Final = get_env_int("MAX_TILE_HEIGHT", 512) +OPENAI_FILE_SEARCH_COST_PER_1K_CALLS: Final = get_env_float("OPENAI_FILE_SEARCH_COST_PER_1K_CALLS", 2.5 / 1000) GROQ_BROWSER_VISIT_WEBSITE_COST_PER_CALL: Final = 1.0 / 1000 # Azure OpenAI Assistants feature costs # Source: https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/ -AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY: Final = float( - os.getenv("AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY", 0.1) # $0.1 USD per 1 GB/Day +AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY: Final = get_env_float("AZURE_FILE_SEARCH_COST_PER_GB_PER_DAY", 0.1) +AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS: Final = get_env_float("AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS", 3.0) +AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS: Final = get_env_float( + "AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS", 12.0 ) -AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS: Final = float( - os.getenv("AZURE_COMPUTER_USE_INPUT_COST_PER_1K_TOKENS", 3.0) # $0.003 USD per 1K Tokens -) -AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS: Final = float( - os.getenv("AZURE_COMPUTER_USE_OUTPUT_COST_PER_1K_TOKENS", 12.0) # $0.012 USD per 1K Tokens -) -AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY: Final = float( - os.getenv("AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY", 0.1) # $0.1 USD per 1 GB/Day (same as file search) -) -MIN_NON_ZERO_TEMPERATURE: Final = float(os.getenv("MIN_NON_ZERO_TEMPERATURE", 0.0001)) +AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY: Final = get_env_float("AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY", 0.1) +MIN_NON_ZERO_TEMPERATURE: Final = get_env_float("MIN_NON_ZERO_TEMPERATURE", 0.0001) #### RELIABILITY #### -REPEATED_STREAMING_CHUNK_LIMIT: Final = int( - os.getenv("REPEATED_STREAMING_CHUNK_LIMIT", 100) +REPEATED_STREAMING_CHUNK_LIMIT: Final = get_env_int( + "REPEATED_STREAMING_CHUNK_LIMIT", 100 ) # catch if model starts looping the same chunk while streaming. Uses high default to prevent false positives. # Shared maxsize for functools.lru_cache usage across hot paths. # Defaulted to 64 to avoid cache thrash in multi-model production workloads. -DEFAULT_MAX_LRU_CACHE_SIZE: Final = int(os.getenv("DEFAULT_MAX_LRU_CACHE_SIZE", 64)) +DEFAULT_MAX_LRU_CACHE_SIZE: Final = get_env_int("DEFAULT_MAX_LRU_CACHE_SIZE", 64) _REALTIME_BODY_CACHE_SIZE = 1000 # Keep realtime helper caches bounded; workloads rarely exceed 1k models/intents -INITIAL_RETRY_DELAY: Final = float(os.getenv("INITIAL_RETRY_DELAY", 0.5)) -MAX_RETRY_DELAY: Final = float(os.getenv("MAX_RETRY_DELAY", 8.0)) -JITTER: Final = float(os.getenv("JITTER", 0.75)) -DEFAULT_IN_MEMORY_TTL = int(os.getenv("DEFAULT_IN_MEMORY_TTL", 5)) # default time to live for the in-memory cache -DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE: Final = int( - os.getenv("DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE", 1000) +INITIAL_RETRY_DELAY: Final = get_env_float("INITIAL_RETRY_DELAY", 0.5) +MAX_RETRY_DELAY: Final = get_env_float("MAX_RETRY_DELAY", 8.0) +JITTER: Final = get_env_float("JITTER", 0.75) +DEFAULT_IN_MEMORY_TTL = get_env_int("DEFAULT_IN_MEMORY_TTL", 5) # default time to live for the in-memory cache +DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE: Final = get_env_int( + "DEFAULT_MAX_REDIS_BATCH_CACHE_SIZE", 1000 ) # default max size for redis batch cache -DEFAULT_POLLING_INTERVAL: Final = float( - os.getenv("DEFAULT_POLLING_INTERVAL", 0.03) +DEFAULT_POLLING_INTERVAL: Final = get_env_float( + "DEFAULT_POLLING_INTERVAL", 0.03 ) # default polling interval for the scheduler -AZURE_OPERATION_POLLING_TIMEOUT: Final = int(os.getenv("AZURE_OPERATION_POLLING_TIMEOUT", 120)) +AZURE_OPERATION_POLLING_TIMEOUT: Final = get_env_int("AZURE_OPERATION_POLLING_TIMEOUT", 120) AZURE_DOCUMENT_INTELLIGENCE_API_VERSION: Final = str(os.getenv("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "2024-11-30")) -AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: Final = int(os.getenv("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", 96)) -REDIS_SOCKET_TIMEOUT: Final = float(os.getenv("REDIS_SOCKET_TIMEOUT", 0.1)) +AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: Final = get_env_int("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", 96) +REDIS_SOCKET_TIMEOUT: Final = get_env_float("REDIS_SOCKET_TIMEOUT", 0.1) CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS: Final[float] = 5.0 -REDIS_CONNECTION_POOL_TIMEOUT: Final = int(os.getenv("REDIS_CONNECTION_POOL_TIMEOUT", 5)) -REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5)) -REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60)) +REDIS_CONNECTION_POOL_TIMEOUT: Final = get_env_int("REDIS_CONNECTION_POOL_TIMEOUT", 5) +REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD: Final = get_env_int("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5) +REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT: Final = get_env_int("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60) REDIS_CIRCUIT_BREAKER_ENABLED: Final = os.getenv("REDIS_CIRCUIT_BREAKER_ENABLED", "true").lower() == "true" # minimum seconds a timeout-only failure streak must span before it can open the breaker, # so one event-loop stall timing out many queued calls at once does not trip it -REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION: Final = float(os.getenv("REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", 5.0)) -REDIS_TIMEOUT_LOG_INTERVAL: Final = float(os.getenv("REDIS_TIMEOUT_LOG_INTERVAL", "5.0")) +REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION: Final = get_env_float("REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION", 5.0) +REDIS_TIMEOUT_LOG_INTERVAL: Final = get_env_float("REDIS_TIMEOUT_LOG_INTERVAL", 5.0) # Seconds of idle before a Redis cluster connection is validated with a PING and # reconnected if dead, so a connection silently dropped by a cluster restart # (e.g. ElastiCache Serverless maintenance) is not reused while broken REDIS_CLUSTER_HEALTH_CHECK_INTERVAL: Final = 25 # Default Redis major version to assume when version cannot be determined # Using 7 as it's the modern version that supports LPOP with count parameter -DEFAULT_REDIS_MAJOR_VERSION: Final = int(os.getenv("DEFAULT_REDIS_MAJOR_VERSION", 7)) -NON_LLM_CONNECTION_TIMEOUT: Final = int( - os.getenv("NON_LLM_CONNECTION_TIMEOUT", 15) +DEFAULT_REDIS_MAJOR_VERSION: Final = get_env_int("DEFAULT_REDIS_MAJOR_VERSION", 7) +NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( + "NON_LLM_CONNECTION_TIMEOUT", 15 ) # timeout for adjacent services (e.g. jwt auth) -MAX_EXCEPTION_MESSAGE_LENGTH: Final = int(os.getenv("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)) -MAX_STRING_LENGTH_PROMPT_IN_DB: Final = int(os.getenv("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)) -BEDROCK_MAX_POLICY_SIZE: Final = int(os.getenv("BEDROCK_MAX_POLICY_SIZE", 75)) +MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) +MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) +BEDROCK_MAX_POLICY_SIZE: Final = get_env_int("BEDROCK_MAX_POLICY_SIZE", 75) # One entry per distinct AWS credential-argument set. Per-user cost attribution passes the attributed # identity as aws_session_name, so this bounds how many attributed identities keep a cached STS session. BEDROCK_IAM_CACHE_MAX_ENTRIES: Final = 1000 @@ -584,27 +579,27 @@ BEDROCK_IAM_CACHE_FETCH_LOCK_STRIPES: Final = 64 # Retire a cached STS credential this many seconds before AWS expires it, so a request that reads it # still has a usable credential for the whole call. STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 -BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = int(os.getenv("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)) +BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 -REPLICATE_POLLING_DELAY_SECONDS: Final = float(os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5)) -DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096)) +REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) +DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 -TOGETHER_AI_4_B: Final = int(os.getenv("TOGETHER_AI_4_B", 4)) -TOGETHER_AI_8_B: Final = int(os.getenv("TOGETHER_AI_8_B", 8)) -TOGETHER_AI_21_B: Final = int(os.getenv("TOGETHER_AI_21_B", 21)) -TOGETHER_AI_41_B: Final = int(os.getenv("TOGETHER_AI_41_B", 41)) -TOGETHER_AI_80_B: Final = int(os.getenv("TOGETHER_AI_80_B", 80)) -TOGETHER_AI_110_B: Final = int(os.getenv("TOGETHER_AI_110_B", 110)) -TOGETHER_AI_EMBEDDING_150_M: Final = int(os.getenv("TOGETHER_AI_EMBEDDING_150_M", 150)) -TOGETHER_AI_EMBEDDING_350_M: Final = int(os.getenv("TOGETHER_AI_EMBEDDING_350_M", 350)) -QDRANT_SCALAR_QUANTILE: Final = float(os.getenv("QDRANT_SCALAR_QUANTILE", 0.99)) -QDRANT_VECTOR_SIZE: Final = int(os.getenv("QDRANT_VECTOR_SIZE", 1536)) -CACHED_STREAMING_CHUNK_DELAY: Final = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0.02)) -AUDIO_SPEECH_CHUNK_SIZE: Final = int( - os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192) +TOGETHER_AI_4_B: Final = get_env_int("TOGETHER_AI_4_B", 4) +TOGETHER_AI_8_B: Final = get_env_int("TOGETHER_AI_8_B", 8) +TOGETHER_AI_21_B: Final = get_env_int("TOGETHER_AI_21_B", 21) +TOGETHER_AI_41_B: Final = get_env_int("TOGETHER_AI_41_B", 41) +TOGETHER_AI_80_B: Final = get_env_int("TOGETHER_AI_80_B", 80) +TOGETHER_AI_110_B: Final = get_env_int("TOGETHER_AI_110_B", 110) +TOGETHER_AI_EMBEDDING_150_M: Final = get_env_int("TOGETHER_AI_EMBEDDING_150_M", 150) +TOGETHER_AI_EMBEDDING_350_M: Final = get_env_int("TOGETHER_AI_EMBEDDING_350_M", 350) +QDRANT_SCALAR_QUANTILE: Final = get_env_float("QDRANT_SCALAR_QUANTILE", 0.99) +QDRANT_VECTOR_SIZE: Final = get_env_int("QDRANT_VECTOR_SIZE", 1536) +CACHED_STREAMING_CHUNK_DELAY: Final = get_env_float("CACHED_STREAMING_CHUNK_DELAY", 0.02) +AUDIO_SPEECH_CHUNK_SIZE: Final = get_env_int( + "AUDIO_SPEECH_CHUNK_SIZE", 8192 ) # chunk_size for audio speech streaming. Balance between latency and memory usage -DEFAULT_MAX_TOKENS_FOR_TRITON: Final = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000)) +DEFAULT_MAX_TOKENS_FOR_TRITON: Final = get_env_int("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000) #### Networking settings #### # Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this # value so longer-running surfaces (Router `timeout or litellm.request_timeout`, @@ -617,12 +612,13 @@ DEFAULT_REQUEST_TIMEOUT_SECONDS: Final[float] = 6000.0 # deadline and connect handshake (see ``http_handler`` cached handler paths). COMPLETION_HTTP_FALLBACK_SECONDS: Final[float] = 600.0 HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: Final[float] = 5.0 -SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS: Final[float] = float( - os.getenv("SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS", "5.0") +SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS: Final[float] = get_env_float("SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS", 5.0) +_request_timeout_from_env: Final = get_env_float_or_none("REQUEST_TIMEOUT") +request_timeout: float = ( + DEFAULT_REQUEST_TIMEOUT_SECONDS if _request_timeout_from_env is None else _request_timeout_from_env ) -request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS)))) -request_timeout_explicitly_set: bool = "REQUEST_TIMEOUT" in os.environ -DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)) # 10 minutes +request_timeout_explicitly_set: bool = _request_timeout_from_env is not None +DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0) # 10 minutes AGENT_KILL_SWITCH_TIMEOUT_SECONDS: Final = 10.0 AGENT_KILL_SWITCH_RESPONSE_BODY_MAX_CHARS: Final = 2000 # Patterns that indicate a localhost/internal URL in A2A agent cards that should be @@ -646,20 +642,20 @@ STREAM_SSE_DATA_PREFIX: Final[str] = "data: " STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n' STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8") ### SPEND TRACKING ### -DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = float( - os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400) +DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = get_env_float( + "DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400 ) # price per second for a100 80GB -FIREWORKS_AI_56_B_MOE: Final = int(os.getenv("FIREWORKS_AI_56_B_MOE", 56)) -FIREWORKS_AI_176_B_MOE: Final = int(os.getenv("FIREWORKS_AI_176_B_MOE", 176)) -FIREWORKS_AI_4_B: Final = int(os.getenv("FIREWORKS_AI_4_B", 4)) -FIREWORKS_AI_16_B: Final = int(os.getenv("FIREWORKS_AI_16_B", 16)) -FIREWORKS_AI_80_B: Final = int(os.getenv("FIREWORKS_AI_80_B", 80)) +FIREWORKS_AI_56_B_MOE: Final = get_env_int("FIREWORKS_AI_56_B_MOE", 56) +FIREWORKS_AI_176_B_MOE: Final = get_env_int("FIREWORKS_AI_176_B_MOE", 176) +FIREWORKS_AI_4_B: Final = get_env_int("FIREWORKS_AI_4_B", 4) +FIREWORKS_AI_16_B: Final = get_env_int("FIREWORKS_AI_16_B", 16) +FIREWORKS_AI_80_B: Final = get_env_int("FIREWORKS_AI_80_B", 80) # https://docs.fireworks.ai/guides/prompt-caching (accessed 2026-09-19): serverless cached prompt tokens # default to a 50% discount off the input rate FIREWORKS_AI_DEFAULT_CACHE_READ_RATE_RATIO: Final = 0.5 #### Logging callback constants #### REDACTED_BY_LITELM_STRING: Final = "REDACTED_BY_LITELM" -MAX_LANGFUSE_INITIALIZED_CLIENTS: Final = int(os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50)) +MAX_LANGFUSE_INITIALIZED_CLIENTS: Final = get_env_int("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50) LANGFUSE_SHUTDOWN_FLUSH_TIMEOUT_MILLIS: Final = 10_000 # Backpressure + lifetime bounds for the /v1/messages streaming relay (see # BaseAnthropicMessagesStreamingIterator.async_sse_wrapper). The relay queue is @@ -667,27 +663,25 @@ LANGFUSE_SHUTDOWN_FLUSH_TIMEOUT_MILLIS: Final = 10_000 # buffer the whole response in memory; the detached-drain cap bounds how many # post-disconnect drains may run concurrently so client behavior can't create # unbounded worker state. -ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE: Final = int( - os.getenv("ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", "1024") +ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE: Final = get_env_int( + "ANTHROPIC_MESSAGES_STREAM_RELAY_QUEUE_MAXSIZE", 1024 ) # Setting this to 0 disables detached draining entirely: every post-disconnect # pump bills whatever partial output it has already collected and aborts the # upstream stream immediately, instead of continuing to drain for the real # terminal usage. -ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS: Final = int( - os.getenv("ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", "100") -) -LOGGING_WORKER_CONCURRENCY: Final = int(os.getenv("LOGGING_WORKER_CONCURRENCY", 100)) # Must be above 0 -LOGGING_WORKER_MAX_QUEUE_SIZE: Final = int(os.getenv("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000)) -LOGGING_WORKER_MAX_TIME_PER_COROUTINE: Final = float(os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0)) +ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS: Final = get_env_int("ANTHROPIC_MESSAGES_MAX_DETACHED_STREAM_DRAINS", 100) +LOGGING_WORKER_CONCURRENCY: Final = get_env_int("LOGGING_WORKER_CONCURRENCY", 100) # Must be above 0 +LOGGING_WORKER_MAX_QUEUE_SIZE: Final = get_env_int("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000) +LOGGING_WORKER_MAX_TIME_PER_COROUTINE: Final = get_env_float("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0) LOGGING_WORKER_TIMEOUT_SUMMARY_WINDOW_SECONDS: Final = 5.0 -LOGGING_WORKER_CLEAR_PERCENTAGE: Final = int( - os.getenv("LOGGING_WORKER_CLEAR_PERCENTAGE", 50) +LOGGING_WORKER_CLEAR_PERCENTAGE: Final = get_env_int( + "LOGGING_WORKER_CLEAR_PERCENTAGE", 50 ) # Percentage of queue to clear (default: 50%) -MAX_ITERATIONS_TO_CLEAR_QUEUE: Final = int(os.getenv("MAX_ITERATIONS_TO_CLEAR_QUEUE", 200)) -MAX_TIME_TO_CLEAR_QUEUE: Final = float(os.getenv("MAX_TIME_TO_CLEAR_QUEUE", 5.0)) -LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS: Final = float( - os.getenv("LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS", 0.5) +MAX_ITERATIONS_TO_CLEAR_QUEUE: Final = get_env_int("MAX_ITERATIONS_TO_CLEAR_QUEUE", 200) +MAX_TIME_TO_CLEAR_QUEUE: Final = get_env_float("MAX_TIME_TO_CLEAR_QUEUE", 5.0) +LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS: Final = get_env_float( + "LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS", 0.5 ) # Cooldown time in seconds before allowing another aggressive clear (default: 0.5s) LOGGING_EXECUTOR_MAX_THREADS: Final = get_env_int("LOGGING_EXECUTOR_MAX_THREADS", 100) LOGGING_EXECUTOR_MAX_PENDING_TASKS: Final = get_env_int("LOGGING_EXECUTOR_MAX_PENDING_TASKS", 10_000) @@ -700,9 +694,9 @@ DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE: Final = os.getenv( LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED: Final = 499 -EMAIL_BUDGET_ALERT_TTL: Final = int(os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60)) # 24 hours in seconds -EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE: Final = float( - os.getenv("EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8) +EMAIL_BUDGET_ALERT_TTL: Final = get_env_int("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60) # 24 hours in seconds +EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE: Final = get_env_float( + "EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8 ) # 80% of max budget ############### LLM Provider Constants ############### ### ANTHROPIC CONSTANTS ### @@ -727,11 +721,11 @@ LITELLM_WEB_SEARCH_TOOL_NAME: Final = "litellm_web_search" DEFAULT_IMAGE_ENDPOINT_MODEL: Final = "dall-e-2" DEFAULT_VIDEO_ENDPOINT_MODEL: Final = "sora-2" -DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS: Final = int(os.getenv("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8)) +DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS: Final = get_env_int("DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS", 8) ### DATAFORSEO CONSTANTS ### -DEFAULT_DATAFORSEO_LOCATION_CODE: Final = int( - os.getenv("DEFAULT_DATAFORSEO_LOCATION_CODE", 2250) +DEFAULT_DATAFORSEO_LOCATION_CODE: Final = get_env_int( + "DEFAULT_DATAFORSEO_LOCATION_CODE", 2250 ) # Default to France (2250) - lower number, commonly used location LITELLM_CHAT_PROVIDERS: Final = [ @@ -1593,20 +1587,20 @@ OPENAI_FINISH_REASONS: Final = [ "tool_calls", "content_filter", ] -HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = int(os.getenv("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60)) # 1 minute +HUMANLOOP_PROMPT_CACHE_TTL_SECONDS: Final = get_env_int("HUMANLOOP_PROMPT_CACHE_TTL_SECONDS", 60) # 1 minute RESPONSE_FORMAT_TOOL_NAME = "json_tool_call" # default tool name used when converting response format to tool call ########################### Logging Callback Constants ########################### AZURE_STORAGE_MSFT_VERSION: Final = "2019-07-07" AZURE_STORAGE_DEFAULT_ENDPOINT_SUFFIX: Final = "core.windows.net" -PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES: Final = int( - os.getenv("PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES", 5) +PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES: Final = get_env_int( + "PROMETHEUS_BUDGET_METRICS_REFRESH_INTERVAL_MINUTES", 5 ) PROMETHEUS_OVERFLOW_SERIES_LABEL_VALUE: Final = "other" PROMETHEUS_ADMITTED_SERIES_FILE_PREFIX: Final = "litellm_admitted_series_" -CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = int(os.getenv("CLOUDZERO_EXPORT_INTERVAL_MINUTES", 60)) +CLOUDZERO_EXPORT_INTERVAL_MINUTES: Final = get_env_int("CLOUDZERO_EXPORT_INTERVAL_MINUTES", 60) MCP_TOOL_NAME_PREFIX: Final = "mcp_tool" -MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = int(os.getenv("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100)) +MAXIMUM_TRACEBACK_LINES_TO_LOG: Final = get_env_int("MAXIMUM_TRACEBACK_LINES_TO_LOG", 100) PASSTHROUGH_UPSTREAM_ERROR_BODY_MAX_LOG_CHARS: Final = 4096 # Headers to control callbacks @@ -1644,11 +1638,11 @@ STANDARD_CUSTOMER_ID_HEADERS: Final = [ "x-litellm-customer-id", "x-litellm-end-user-id", ] -MAX_SPENDLOG_ROWS_TO_QUERY: Final = int( - os.getenv("MAX_SPENDLOG_ROWS_TO_QUERY", 1_000_000) +MAX_SPENDLOG_ROWS_TO_QUERY: Final = get_env_int( + "MAX_SPENDLOG_ROWS_TO_QUERY", 1_000_000 ) # if spendLogs has more than 1M rows, do not query the DB -DEFAULT_SOFT_BUDGET: Final = float( - os.getenv("DEFAULT_SOFT_BUDGET", 50.0) +DEFAULT_SOFT_BUDGET: Final = get_env_float( + "DEFAULT_SOFT_BUDGET", 50.0 ) # by default all litellm proxy keys have a soft budget of 50.0 # makes it clear this is a rate limit error for a litellm virtual key RATE_LIMIT_ERROR_MESSAGE_FOR_VIRTUAL_KEY: Final = "LiteLLM Virtual Key user_api_key_hash" @@ -1715,38 +1709,16 @@ TRANSCRIBE_MEDIA_FETCH_ATTEMPTS: Final = 3 TRANSCRIBE_MEDIA_LAST_MODIFIED_TOLERANCE_SECONDS: Final = 1.0 # S3 Last-Modified carries whole seconds only TRANSCRIBE_MEASURABLE_MEDIA_FORMATS: Final = frozenset({"flac", "mp3", "ogg", "wav"}) # what libsndfile can read -BATCH_STATUS_POLL_INTERVAL_SECONDS: Final = int(os.getenv("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600)) # 1 hour -BATCH_STATUS_POLL_MAX_ATTEMPTS: Final = int(os.getenv("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24)) # for 24 hours +BATCH_STATUS_POLL_INTERVAL_SECONDS: Final = get_env_int("BATCH_STATUS_POLL_INTERVAL_SECONDS", 3600) # 1 hour +BATCH_STATUS_POLL_MAX_ATTEMPTS: Final = get_env_int("BATCH_STATUS_POLL_MAX_ATTEMPTS", 24) # for 24 hours BATCH_TPD_WINDOW_SECONDS: Final = 86400 BATCH_TPD_DESCRIPTOR_SUFFIX: Final = "_tpd" -HEALTH_CHECK_TIMEOUT_SECONDS: Final = int(os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)) # 60 seconds -_background_health_check_max_tokens_env: Final = os.getenv("BACKGROUND_HEALTH_CHECK_MAX_TOKENS") -try: - _raw_background_health_check_max_tokens: Final = ( - _background_health_check_max_tokens_env.strip() if _background_health_check_max_tokens_env is not None else "" - ) - BACKGROUND_HEALTH_CHECK_MAX_TOKENS: int | None = ( - int(_raw_background_health_check_max_tokens) if _raw_background_health_check_max_tokens else None - ) -except (ValueError, TypeError): - BACKGROUND_HEALTH_CHECK_MAX_TOKENS = None - - -_background_health_check_max_tokens_reasoning_env: Final = os.getenv("BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING") -try: - _raw_background_health_check_max_tokens_reasoning: Final = ( - _background_health_check_max_tokens_reasoning_env.strip() - if _background_health_check_max_tokens_reasoning_env is not None - else "" - ) - BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING: int | None = ( - int(_raw_background_health_check_max_tokens_reasoning) - if _raw_background_health_check_max_tokens_reasoning - else None - ) -except (ValueError, TypeError): - BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING = None +HEALTH_CHECK_TIMEOUT_SECONDS: Final = get_env_int("HEALTH_CHECK_TIMEOUT_SECONDS", 60) # 60 seconds +BACKGROUND_HEALTH_CHECK_MAX_TOKENS: int | None = get_env_int_or_none("BACKGROUND_HEALTH_CHECK_MAX_TOKENS") +BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING: int | None = get_env_int_or_none( + "BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING" +) LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME: Final = "litellm-internal-health-check" LITTELM_CLI_SERVICE_ACCOUNT_NAME: Final = "litellm-cli" @@ -1771,22 +1743,22 @@ OTEL_SERVICE_NAME_METADATA_KEYS: Final = ("otel_service_name_override", "otel_se # Key Rotation Constants LITELLM_KEY_ROTATION_ENABLED: Final = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false") -LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS: Final = int( - os.getenv("LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS", 86400) +LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS: Final = get_env_int( + "LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS", 86400 ) # 24 hours default LITELLM_KEY_ROTATION_GRACE_PERIOD: Final[str] = os.getenv( "LITELLM_KEY_ROTATION_GRACE_PERIOD", "" ) # Duration to keep old key valid after rotation (e.g. "24h", "2d"); empty = immediate revoke (default) -LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS: Final = int( - os.getenv("LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS", 600) +LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS: Final = get_env_int( + "LITELLM_KEY_ROTATION_LOCK_TTL_SECONDS", 600 ) # 10 minutes default — caps the deadlock window if a pod crashes mid-rotation UI_SESSION_TOKEN_TEAM_ID: Final = "litellm-dashboard" LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED = os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_ENABLED", "false") -LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS: Final = int( - os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS", 86400) +LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS: Final = get_env_int( + "LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_INTERVAL_SECONDS", 86400 ) # 24 hours default -LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE: Final = int( - os.getenv("LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE", 1000) +LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE: Final = get_env_int( + "LITELLM_EXPIRED_UI_SESSION_KEY_CLEANUP_BATCH_SIZE", 1000 ) LOGIN_THROTTLE_CACHE_KEY_PREFIX: Final = "login_fail" LOGIN_THROTTLE_UNKNOWN_SOURCE: Final = "unknown" @@ -1796,7 +1768,7 @@ LOGIN_THROTTLE_NOT_BLOCKED: Final = (0, 0) LITELLM_PROXY_ADMIN_NAME: Final = "default_user_id" LITELLM_PROXY_BUDGET_NAME: Final = "litellm-proxy-budget" GLOBAL_PROXY_SPEND_CACHE_KEY: Final = f"{LITELLM_PROXY_ADMIN_NAME}:spend" -LITELLM_EXECUTED_BATCH_CONCURRENCY: Final = max(1, int(os.getenv("LITELLM_EXECUTED_BATCH_CONCURRENCY", "4"))) +LITELLM_EXECUTED_BATCH_CONCURRENCY: Final = max(1, get_env_int("LITELLM_EXECUTED_BATCH_CONCURRENCY", 4)) ########################### CLI SSO AUTHENTICATION CONSTANTS ########################### LITELLM_CLI_SOURCE_IDENTIFIER: Final = "litellm-cli" @@ -1805,8 +1777,8 @@ CLI_SSO_SESSION_CACHE_KEY_PREFIX: Final = "cli_sso_session" CLI_SSO_SESSION_TTL_SECONDS: Final = 600 CLI_SESSION_KEY_PREFIX: Final = "cli-session" # Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility -CLI_JWT_EXPIRATION_HOURS: Final = int( - os.getenv("CLI_JWT_EXPIRATION_HOURS") or os.getenv("LITELLM_CLI_JWT_EXPIRATION_HOURS") or 24 +CLI_JWT_EXPIRATION_HOURS: Final = get_env_int( + "CLI_JWT_EXPIRATION_HOURS", get_env_int("LITELLM_CLI_JWT_EXPIRATION_HOURS", 24) ) # Comma-separated allowlisted OIDC claim map for CLI SSO polling, e.g. # "employment_type->acme_employment_type,org_info.department->department" @@ -1824,7 +1796,7 @@ DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME: Final = "db_daily_tag_spend_update_job" PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME: Final = "prometheus_emit_budget_metrics" CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data" MAVVRIK_FOCUS_EXPORT_JOB_NAME: Final = "mavvrik_focus_export_usage_data" -CLOUDZERO_MAX_FETCHED_DATA_RECORDS: Final = int(os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)) +CLOUDZERO_MAX_FETCHED_DATA_RECORDS: Final = get_env_int("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000) SPEND_LOG_CLEANUP_JOB_NAME: Final = "spend_log_cleanup" BACKGROUND_HEALTH_CHECK_DB_SAVE_JOB_NAME: Final = "background_health_check_db_save" KEY_ROTATION_JOB_NAME: Final = "litellm_key_rotation_job" @@ -1835,20 +1807,20 @@ USER_SPEND_ALERTS_JOB_ID: Final = "user_spend_alerts_job" PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job" SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report" SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning" -SPEND_LOG_RUN_LOOPS: Final = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500)) -SPEND_LOG_CLEANUP_BATCH_SIZE: Final = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000)) -SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)) -SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = float( - os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5) +SPEND_LOG_RUN_LOOPS: Final = get_env_int("SPEND_LOG_RUN_LOOPS", 500) +SPEND_LOG_CLEANUP_BATCH_SIZE: Final = get_env_int("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000) +SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = get_env_int("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3) +SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = get_env_float( + "SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5 ) -SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", "300")) -SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", "30")) -SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = int(os.getenv("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", "100000")) -SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = float( - os.getenv("SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", "5") +SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", 300.0) +SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = get_env_float("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", 30.0) +SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = get_env_int("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", 100000) +SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS: Final = get_env_float( + "SCHEDULED_JOB_SHUTDOWN_FINISH_TIMEOUT_SECONDS", 5.0 ) -SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS: Final = float( - os.getenv("SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS", "5") +SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS: Final = get_env_float( + "SCHEDULED_JOB_SHUTDOWN_CANCEL_TIMEOUT_SECONDS", 5.0 ) TOOL_SPEND_TOP_TOOLS: Final = 100 MODEL_INSIGHTS_TOP_MODELS: Final = 10 @@ -1856,54 +1828,54 @@ MODEL_INSIGHTS_MAX_RANGE_DAYS: Final = 365 MODEL_INSIGHTS_DEFAULT_TASK: Final = "uncategorized" MODEL_INSIGHTS_TASK_TAG_PREFIX: Final = "task:" SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day") -SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7)) -SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000))) -SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_ROWS", "100"))) -SPEND_ROLLUP_LOCK_TIMEOUT_MS: Final = max(1, int(os.getenv("SPEND_ROLLUP_LOCK_TIMEOUT_MS", "5000"))) -SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100)) -SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000"))) -SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0)) -RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, int(os.getenv("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", "3"))) -RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = float(os.getenv("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", "0.2")) -SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000)) -PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, int(os.getenv("PROXY_DB_LOOKUP_MAX_CONCURRENCY", "25"))) -PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, float(os.getenv("PROXY_DB_LOOKUP_DEADLINE_SECONDS", "10"))) -PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, float(os.getenv("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", "30"))) -DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)) # 1 minute -PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597)) -RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_BATCH_SIZE", "500"))) -RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN", "100"))) +SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = get_env_int("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7) +SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, get_env_int("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)) +SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, get_env_int("SPEND_LOG_WRITE_BATCH_MAX_ROWS", 100)) +SPEND_ROLLUP_LOCK_TIMEOUT_MS: Final = max(1, get_env_int("SPEND_ROLLUP_LOCK_TIMEOUT_MS", 5000)) +SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = get_env_int("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100) +SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, get_env_int("SPEND_LOG_QUEUE_MAX_BYTES", 64000000)) +SPEND_LOG_QUEUE_POLL_INTERVAL: Final = get_env_float("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0) +RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, get_env_int("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", 3)) +RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = get_env_float("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", 0.2) +SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = get_env_int("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000) +PROXY_DB_LOOKUP_MAX_CONCURRENCY: Final = max(1, get_env_int("PROXY_DB_LOOKUP_MAX_CONCURRENCY", 25)) +PROXY_DB_LOOKUP_DEADLINE_SECONDS: Final = max(0.1, get_env_float("PROXY_DB_LOOKUP_DEADLINE_SECONDS", 10.0)) +PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS: Final = max(0.0, get_env_float("PROXY_DB_LOOKUP_STALL_WINDOW_SECONDS", 30.0)) +DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = get_env_int("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60) # 1 minute +PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597) +RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, get_env_int("RESET_BUDGET_JOB_BATCH_SIZE", 500)) +RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN: Final = max(1, get_env_int("RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN", 100)) RESET_BUDGET_JOB_NAME: Final = "reset_budget_job" # Comfortably longer than one PROXY_BUDGET_RESCHEDULER_MIN_TIME tick, so a healthy # leader keeps the lease across its own run, and a crashed one strands the sweep for # at most a single tick. RESET_BUDGET_JOB_LOCK_TTL_SECONDS: Final[int] = 900 -PROXY_BATCH_POLLING_INTERVAL: Final = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 3600)) -MAX_OBJECTS_PER_POLL_CYCLE: Final = max(1, int(os.getenv("MAX_OBJECTS_PER_POLL_CYCLE", 50))) -MANAGED_OBJECT_STALENESS_CUTOFF_DAYS: Final = max(1, int(os.getenv("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7))) +PROXY_BATCH_POLLING_INTERVAL: Final = get_env_int("PROXY_BATCH_POLLING_INTERVAL", 3600) +MAX_OBJECTS_PER_POLL_CYCLE: Final = max(1, get_env_int("MAX_OBJECTS_PER_POLL_CYCLE", 50)) +MANAGED_OBJECT_STALENESS_CUTOFF_DAYS: Final = max(1, get_env_int("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7)) BATCH_OUTPUT_FILE_LOOKUP_TIMEOUT_SECONDS: Final = 10.0 BATCH_OUTPUT_FILE_FALLBACK_RETRY_AFTER_SECONDS: Final = 60 -STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, int(os.getenv("STALE_OBJECT_CLEANUP_BATCH_SIZE", 1000))) +STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, get_env_int("STALE_OBJECT_CLEANUP_BATCH_SIZE", 1000)) # Set PROXY_BATCH_POLLING_ENABLED=false to disable the CheckBatchCost and # CheckResponsesCost background polling jobs entirely (e.g. to avoid DB load on # installations with large numbers of stale managed objects). _batch_polling_env: Final = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower() PROXY_BATCH_POLLING_ENABLED: Final = _batch_polling_env == "true" -BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS: Final = float( - os.getenv("BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", "5") +BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS: Final = get_env_float( + "BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", 5.0 ) -BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = float( - os.getenv("BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", "60") +BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = get_env_float( + "BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", 60.0 ) -BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = float( - os.getenv("BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", "3600") +BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = get_env_float( + "BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", 3600.0 ) _background_interaction_cost_polling_env: Final = os.getenv( "BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true" ).lower() BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true" -PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)) -PROXY_BATCH_WRITE_AT: Final = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds, increased from 10 +PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605) +PROXY_BATCH_WRITE_AT: Final = get_env_int("PROXY_BATCH_WRITE_AT", 10) # in seconds, increased from 10 PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30) # APScheduler Configuration - MEMORY LEAK FIX @@ -1912,10 +1884,10 @@ APSCHEDULER_COALESCE: Final = os.getenv("APSCHEDULER_COALESCE", "True").lower() "true", "1", ] # collapse many missed runs into one -APSCHEDULER_MISFIRE_GRACE_TIME: Final = int( - os.getenv("APSCHEDULER_MISFIRE_GRACE_TIME", 3600) +APSCHEDULER_MISFIRE_GRACE_TIME: Final = get_env_int( + "APSCHEDULER_MISFIRE_GRACE_TIME", 3600 ) # ignore runs older than 1 hour (was 120) -APSCHEDULER_MAX_INSTANCES: Final = int(os.getenv("APSCHEDULER_MAX_INSTANCES", 1)) # prevent concurrent job instances +APSCHEDULER_MAX_INSTANCES: Final = get_env_int("APSCHEDULER_MAX_INSTANCES", 1) # prevent concurrent job instances APSCHEDULER_REPLACE_EXISTING: Final = os.getenv("APSCHEDULER_REPLACE_EXISTING", "True").lower() in [ "true", "1", @@ -1929,25 +1901,25 @@ DEFAULT_STAGGER_WINDOW_SECONDS: Final = 300 # This will run tag spcific tasks at a later time to smooth QPS DAILY_TAG_SPEND_BATCH_MULTIPLIER: Final = 2.3 -DEFAULT_HEALTH_CHECK_INTERVAL: Final = int(os.getenv("DEFAULT_HEALTH_CHECK_INTERVAL", 300)) # 5 minutes -DEFAULT_SHARED_HEALTH_CHECK_TTL: Final = int( - os.getenv("DEFAULT_SHARED_HEALTH_CHECK_TTL", 300) +DEFAULT_HEALTH_CHECK_INTERVAL: Final = get_env_int("DEFAULT_HEALTH_CHECK_INTERVAL", 300) # 5 minutes +DEFAULT_SHARED_HEALTH_CHECK_TTL: Final = get_env_int( + "DEFAULT_SHARED_HEALTH_CHECK_TTL", 300 ) # 5 minutes - TTL for cached health check results -DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL: Final = int( - os.getenv("DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL", 60) +DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL: Final = get_env_int( + "DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL", 60 ) # 1 minute - TTL for health check lock DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER: Final = 2 # health state is stale after interval * this -PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS: Final = int(os.getenv("PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS", 9)) -DEFAULT_MODEL_CREATED_AT_TIME: Final = int( - os.getenv("DEFAULT_MODEL_CREATED_AT_TIME", 1677610602) +PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS: Final = get_env_int("PROMETHEUS_FALLBACK_STATS_SEND_TIME_HOURS", 9) +DEFAULT_MODEL_CREATED_AT_TIME: Final = get_env_int( + "DEFAULT_MODEL_CREATED_AT_TIME", 1677610602 ) # returns on `/models` endpoint -DEFAULT_SLACK_ALERTING_THRESHOLD: Final = int(os.getenv("DEFAULT_SLACK_ALERTING_THRESHOLD", 300)) -MAX_TEAM_LIST_LIMIT: Final = int(os.getenv("MAX_TEAM_LIST_LIMIT", 20)) -MAX_POLICY_ESTIMATE_IMPACT_ROWS: Final = int(os.getenv("MAX_POLICY_ESTIMATE_IMPACT_ROWS", 1000)) -DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7)) -LENGTH_OF_LITELLM_GENERATED_KEY: Final = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16)) -MINIMUM_CUSTOM_KEY_LENGTH: Final = int(os.getenv("MINIMUM_CUSTOM_KEY_LENGTH", 16)) -SECRET_MANAGER_REFRESH_INTERVAL: Final = int(os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400)) +DEFAULT_SLACK_ALERTING_THRESHOLD: Final = get_env_int("DEFAULT_SLACK_ALERTING_THRESHOLD", 300) +MAX_TEAM_LIST_LIMIT: Final = get_env_int("MAX_TEAM_LIST_LIMIT", 20) +MAX_POLICY_ESTIMATE_IMPACT_ROWS: Final = get_env_int("MAX_POLICY_ESTIMATE_IMPACT_ROWS", 1000) +DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = get_env_float("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7) +LENGTH_OF_LITELLM_GENERATED_KEY: Final = get_env_int("LENGTH_OF_LITELLM_GENERATED_KEY", 16) +MINIMUM_CUSTOM_KEY_LENGTH: Final = get_env_int("MINIMUM_CUSTOM_KEY_LENGTH", 16) +SECRET_MANAGER_REFRESH_INTERVAL: Final = get_env_int("SECRET_MANAGER_REFRESH_INTERVAL", 86400) OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS: Final = frozenset({"openai", "azure"}) LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "default_internal_user_params", @@ -1980,9 +1952,9 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "aws_sqs_callback_params", ] SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"] -DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)) +DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = get_env_int("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60) AUTH_OBJECTS_TARGET: Final = "auth_objects" -DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = int(os.getenv("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)) +DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = get_env_int("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600) SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600 SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30 SPEND_LOG_KEY_METADATA_CACHE_MAX_ITEMS: Final = 10000 @@ -2090,14 +2062,14 @@ SENTRY_PII_DENYLIST: Final = [ ] # CoroutineChecker cache configuration -COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY: Final = int(os.getenv("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000)) +COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY: Final = get_env_int("COROUTINE_CHECKER_MAX_SIZE_IN_MEMORY", 1000) ########################### RAG Text Splitter Constants ########################### -DEFAULT_CHUNK_SIZE: Final = int(os.getenv("DEFAULT_CHUNK_SIZE", 1000)) -DEFAULT_CHUNK_OVERLAP: Final = int(os.getenv("DEFAULT_CHUNK_OVERLAP", 200)) +DEFAULT_CHUNK_SIZE: Final = get_env_int("DEFAULT_CHUNK_SIZE", 1000) +DEFAULT_CHUNK_OVERLAP: Final = get_env_int("DEFAULT_CHUNK_OVERLAP", 200) ########################### S3 Vectors RAG Constants ########################### -S3_VECTORS_DEFAULT_DIMENSION: Final = int(os.getenv("S3_VECTORS_DEFAULT_DIMENSION", 1024)) +S3_VECTORS_DEFAULT_DIMENSION: Final = get_env_int("S3_VECTORS_DEFAULT_DIMENSION", 1024) S3_VECTORS_DEFAULT_DISTANCE_METRIC: Final = str(os.getenv("S3_VECTORS_DEFAULT_DISTANCE_METRIC", "cosine")) S3_VECTORS_DEFAULT_NON_FILTERABLE_METADATA_KEYS: Final = ["source_text"] @@ -2110,11 +2082,11 @@ MICROSOFT_USER_LAST_NAME_ATTRIBUTE: Final = str(os.getenv("MICROSOFT_USER_LAST_N # Maximum payload size (in bytes) to fully serialize for DEBUG logging. # Payloads larger than this are truncated to avoid multi-second json.dumps blocking the response. -MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG: Final = int(os.getenv("MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG", 102400)) # 100 KB +MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG: Final = get_env_int("MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG", 102400) # 100 KB # Policy template enrichment -MAX_COMPETITOR_NAMES: Final = int(os.getenv("MAX_COMPETITOR_NAMES", 100)) -COMPETITOR_LLM_TEMPERATURE: Final = float(os.getenv("COMPETITOR_LLM_TEMPERATURE", 0.3)) +MAX_COMPETITOR_NAMES: Final = get_env_int("MAX_COMPETITOR_NAMES", 100) +COMPETITOR_LLM_TEMPERATURE: Final = get_env_float("COMPETITOR_LLM_TEMPERATURE", 0.3) DEFAULT_COMPETITOR_DISCOVERY_MODEL: Final = "gpt-4o-mini" # Advisor tool orchestration diff --git a/litellm/litellm_core_utils/env_utils.py b/litellm/litellm_core_utils/env_utils.py index d641884b4cd..6883af98bbd 100644 --- a/litellm/litellm_core_utils/env_utils.py +++ b/litellm/litellm_core_utils/env_utils.py @@ -4,23 +4,65 @@ Utility helpers for reading and parsing environment variables. import logging import os -from typing import Final +from typing import Final, TypeVar + +_NumberT: Final = TypeVar("_NumberT", int, float) + +# Marked before logging: the LiteLLM loggers' own filters read env knobs through these helpers, so +# warning first would re-enter the helper from inside the log call and recurse until RecursionError. +_reported_unparsable_values: Final[set[tuple[str, str]]] = set() # mutable-ok: process-wide warn-once registry + + +def _warn_unparsable_once(env_var: str, raw: str, number_type: type[int] | type[float]) -> None: + key: Final = (env_var, raw) + if key in _reported_unparsable_values: + return + _reported_unparsable_values.add(key) + logging.getLogger("LiteLLM").warning( + "%s=%r is not a valid %s. Treating it as unset.", env_var, raw, number_type.__name__ + ) + + +def _get_env_number_or_none(env_var: str, number_type: type[_NumberT]) -> _NumberT | None: + raw: Final = os.getenv(env_var) + if raw is None or not raw.strip(): + return None + try: + return number_type(raw.strip()) + except ValueError: + _warn_unparsable_once(env_var, raw, number_type) + return None + + +def get_env_int_or_none(env_var: str) -> int | None: + """Parse an environment variable as an integer, returning None when it is unset, blank or unparsable. + + Use this instead of `get_env_int` when callers must distinguish "explicitly configured" + from "left at the default", for example when an override should take precedence over a + value resolved from somewhere else. + """ + return _get_env_number_or_none(env_var, int) + + +def get_env_float_or_none(env_var: str) -> float | None: + """Float counterpart of `get_env_int_or_none`.""" + return _get_env_number_or_none(env_var, float) def get_env_int(env_var: str, default: int) -> int: - """Parse an environment variable as an integer, falling back to default on invalid values. + """Parse an environment variable as an integer, falling back to default when it is unset, blank or unparsable. - Handles empty strings, whitespace, and non-numeric values gracefully - so that misconfiguration doesn't crash the process at import time. + An unparsable value is warned about once, so misconfiguration neither crashes the process at + import time nor goes unnoticed. """ - raw = os.getenv(env_var) - if raw is None: - return default - raw = raw.strip() - try: - return int(raw) - except (ValueError, TypeError): - return default + value: Final = get_env_int_or_none(env_var) + return default if value is None else value + + +def get_env_float(env_var: str, default: float) -> float: + """Float counterpart of `get_env_int`.""" + value: Final = get_env_float_or_none(env_var) + return default if value is None else value def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int) -> int: @@ -41,19 +83,3 @@ def get_env_int_in_range(env_var: str, default: int, minimum: int, maximum: int) default, ) return default - - -def get_env_int_or_none(env_var: str) -> int | None: - """Parse an environment variable as an integer, returning None when it is unset or unusable. - - Use this instead of `get_env_int` when callers must distinguish "explicitly configured" - from "left at the default", for example when an override should take precedence over a - value resolved from somewhere else. - """ - raw: Final = os.getenv(env_var) - if raw is None: - return None - try: - return int(raw.strip()) - except (ValueError, TypeError): - return None diff --git a/litellm/proxy/agent_endpoints/endpoints.py b/litellm/proxy/agent_endpoints/endpoints.py index 7545a1117a1..89794fc2480 100644 --- a/litellm/proxy/agent_endpoints/endpoints.py +++ b/litellm/proxy/agent_endpoints/endpoints.py @@ -21,6 +21,7 @@ from typing_extensions import ReadOnly, Required, assert_never import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS, AGENT_HEALTH_CHECK_TIMEOUT_SECONDS from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._types import ( CommonProxyErrors, @@ -228,10 +229,6 @@ def _check_agent_management_permission(user_api_key_dict: UserAPIKeyAuth) -> Non ) -AGENT_HEALTH_CHECK_TIMEOUT_SECONDS: Final = float(os.environ.get("LITELLM_AGENT_HEALTH_CHECK_TIMEOUT", "5.0")) -AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS = float(os.environ.get("LITELLM_AGENT_HEALTH_CHECK_GATHER_TIMEOUT", "30.0")) - - class _AgentHealthResult(TypedDict, total=False): agent_id: Required[str] healthy: Required[bool] diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index cf90d90ea33..20504982fde 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -55,6 +55,7 @@ from litellm import _custom_logger_compatible_callbacks_literal from litellm.constants import ( DEFAULT_MODEL_CREATED_AT_TIME, FILE_USAGE_MAX_TRACKED_COUNTERS, + LITELLM_CONFIG_CACHE_TTL_SECONDS, LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL, MAX_TEAM_LIST_LIMIT, PROXY_REJECTED_BEFORE_ROUTING_KEY, @@ -150,6 +151,7 @@ from litellm.litellm_core_utils.core_helpers import ( is_expected_client_error, ) from litellm.litellm_core_utils.duration_parser import get_budget_window_start +from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -4505,7 +4507,6 @@ async def _lookup_deprecated_key( # DualCache for LiteLLM_Config param_name reads. # Redis layer is attached in proxy_server._init_cache. -LITELLM_CONFIG_CACHE_TTL_SECONDS: Final[int] = int(os.environ.get("LITELLM_CONFIG_PARAM_CACHE_TTL_SECONDS", "60")) _CONFIG_CACHE_MISS: Final[str] = "__litellm_config_param_miss__" litellm_config_cache: Final[DualCache] = DualCache( @@ -4789,28 +4790,28 @@ class PrismaClient: self._db_health_watchdog_task: asyncio.Task | None = None self._view_setup_task: asyncio.Task[_ViewSetupOutcome] | None = None self._db_last_reconnect_attempt_ts: float = 0.0 - self._db_reconnect_cooldown_seconds: int = max(1, int(os.getenv("PRISMA_RECONNECT_COOLDOWN_SECONDS", "15"))) + self._db_reconnect_cooldown_seconds: int = max(1, get_env_int("PRISMA_RECONNECT_COOLDOWN_SECONDS", 15)) self._db_read_only_recreate_ts: float = 0.0 self._db_read_only_recreate_streak: int = 0 self._db_health_watchdog_interval_seconds: int = max( - 5, int(os.getenv("PRISMA_HEALTH_WATCHDOG_INTERVAL_SECONDS", "30")) + 5, get_env_int("PRISMA_HEALTH_WATCHDOG_INTERVAL_SECONDS", 30) ) self._db_health_watchdog_enabled: bool = ( str_to_bool(os.getenv("PRISMA_HEALTH_WATCHDOG_ENABLED", "true")) is True ) self._db_health_watchdog_probe_timeout_seconds: float = max( 0.5, - float(os.getenv("PRISMA_HEALTH_WATCHDOG_PROBE_TIMEOUT_SECONDS", "5.0")), + get_env_float("PRISMA_HEALTH_WATCHDOG_PROBE_TIMEOUT_SECONDS", 5.0), ) self._db_watchdog_reconnect_timeout_seconds: float = max( - 1.0, float(os.getenv("PRISMA_WATCHDOG_RECONNECT_TIMEOUT_SECONDS", "30.0")) + 1.0, get_env_float("PRISMA_WATCHDOG_RECONNECT_TIMEOUT_SECONDS", 30.0) ) self._db_auth_reconnect_timeout_seconds: float = max( - 0.5, float(os.getenv("PRISMA_AUTH_RECONNECT_TIMEOUT_SECONDS", "2.0")) + 0.5, get_env_float("PRISMA_AUTH_RECONNECT_TIMEOUT_SECONDS", 2.0) ) self._db_auth_reconnect_lock_timeout_seconds: float = max( 0.0, - float(os.getenv("PRISMA_AUTH_RECONNECT_LOCK_TIMEOUT_SECONDS", "0.1")), + get_env_float("PRISMA_AUTH_RECONNECT_LOCK_TIMEOUT_SECONDS", 0.1), ) self._consecutive_reconnect_failures: int = 0 # Last generation of each read engine whose repair was attempted and @@ -4821,7 +4822,7 @@ class PrismaClient: # back to a caller whose engine is still unrepaired. Bounded at two # entries: a client has one writer and at most one reader. self._failed_recreate_generations: Mapping[PrismaWrapper, int] = MappingProxyType({}) - self._reconnect_escalation_threshold: int = max(1, int(os.getenv("PRISMA_RECONNECT_ESCALATION_THRESHOLD", "3"))) + self._reconnect_escalation_threshold: int = max(1, get_env_int("PRISMA_RECONNECT_ESCALATION_THRESHOLD", 3)) self._engine_pidfd: int = -1 self._engine_pid: int = 0 self._watching_engine: bool = False diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index e65a0dabd6d..6c86b058f0f 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -5,6 +5,7 @@ from types import MappingProxyType from typing import Final, Literal, cast from pydantic import ConfigDict, Field, field_validator, model_validator +from pydantic.fields import FieldInfo from typing_extensions import ReadOnly, Required, TypedDict from litellm._logging import verbose_logger @@ -1000,6 +1001,16 @@ def runtime_stream_scope( LoggingOnlyScope = Literal["input", "output"] +def _is_blank(value: object) -> bool: + return isinstance(value, str) and not value.strip() + + +def _default_if_blank(field: FieldInfo | None, value: object) -> object: + if field is None or field.is_required() or not _is_blank(value): + return value + return field.get_default(call_default_factory=True) + + class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch update guardrails api_key: str | None = Field(default=None, description="API key for the guardrail service") api_base: str | None = Field(default=None, description="Base URL for the guardrail service API") @@ -1288,6 +1299,14 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up ), ) + @model_validator(mode="before") # pyright: ignore[reportUntypedFunctionDecorator] # pydantic decorators are untyped + @classmethod + def blank_values_mean_field_default(cls, data: object) -> object: + if not isinstance(data, Mapping): + return data + fields: Final = cast(Mapping[str, object], data) # cast-ok: pydantic hands a before-validator the raw mapping + return {key: _default_if_blank(cls.model_fields.get(key), value) for key, value in fields.items()} + @field_validator( "mode", "default_action", @@ -1398,14 +1417,14 @@ class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # o ) return self - def __init__(self, **kwargs) -> None: + def __init__(self, **kwargs: object) -> None: default_on: Final = kwargs.pop("default_on", None) - if default_on is not None: + if default_on is not None and not _is_blank(default_on): kwargs["default_on"] = default_on else: kwargs["default_on"] = False - super().__init__(**kwargs) + super().__init__(**kwargs) # pyright: ignore[reportArgumentType] # pydantic validates each field's type at runtime def __contains__(self, key) -> bool: # Define custom behavior for the 'in' operator diff --git a/tests/unit/litellm_core_utils/test_env_utils.py b/tests/unit/litellm_core_utils/test_env_utils.py new file mode 100644 index 00000000000..1ddd172e274 --- /dev/null +++ b/tests/unit/litellm_core_utils/test_env_utils.py @@ -0,0 +1,96 @@ +import logging +from collections.abc import Callable, Iterator + +import pytest + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.env_utils import ( + get_env_float, + get_env_float_or_none, + get_env_int, + get_env_int_or_none, +) + + +@pytest.fixture +def litellm_log(caplog: pytest.LogCaptureFixture) -> Iterator[pytest.LogCaptureFixture]: + verbose_logger.addHandler(caplog.handler) + try: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + yield caplog + finally: + verbose_logger.removeHandler(caplog.handler) + + +def _logged_messages(caplog: pytest.LogCaptureFixture) -> list[str]: + return [record.getMessage() for record in dict.fromkeys(caplog.records)] + + +@pytest.mark.parametrize("raw", ["", " ", "\t"]) +def test_blank_numeric_env_value_means_unset_without_warning( + monkeypatch: pytest.MonkeyPatch, litellm_log: pytest.LogCaptureFixture, raw: str +) -> None: + monkeypatch.setenv("ENV_UTILS_TEST_KNOB", raw) + + assert get_env_float("ENV_UTILS_TEST_KNOB", 1.5) == 1.5 + assert get_env_int("ENV_UTILS_TEST_KNOB", 7) == 7 + assert get_env_float_or_none("ENV_UTILS_TEST_KNOB") is None + assert get_env_int_or_none("ENV_UTILS_TEST_KNOB") is None + assert _logged_messages(litellm_log) == [] + + +def test_set_numeric_env_value_is_parsed(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ENV_UTILS_TEST_FLOAT", " 2.5 ") + monkeypatch.setenv("ENV_UTILS_TEST_INT", " 12 ") + + assert get_env_float("ENV_UTILS_TEST_FLOAT", 1.5) == 2.5 + assert get_env_int("ENV_UTILS_TEST_INT", 7) == 12 + assert get_env_float_or_none("ENV_UTILS_TEST_FLOAT") == 2.5 + assert get_env_int_or_none("ENV_UTILS_TEST_INT") == 12 + + +@pytest.mark.parametrize( + ("read", "raw", "expected"), + [ + (lambda name: get_env_int(name, 7), "fast-int", 7), + (lambda name: get_env_float(name, 1.5), "fast-float", 1.5), + (get_env_int_or_none, "fast-int-or-none", None), + (get_env_float_or_none, "fast-float-or-none", None), + ], +) +def test_unparsable_numeric_env_value_falls_back_and_warns_once( + monkeypatch: pytest.MonkeyPatch, + litellm_log: pytest.LogCaptureFixture, + read: Callable[[str], float | None], + raw: str, + expected: float | None, +) -> None: + monkeypatch.setenv("ENV_UTILS_TEST_KNOB", raw) + + assert read("ENV_UTILS_TEST_KNOB") == expected + assert read("ENV_UTILS_TEST_KNOB") == expected + + warnings = [message for message in _logged_messages(litellm_log) if raw in message] + assert len(warnings) == 1 + assert "ENV_UTILS_TEST_KNOB" in warnings[0] + + +@pytest.mark.parametrize("env_var", ["MAX_STRING_LENGTH_STDOUT_LOG", "MAX_BASE64_LENGTH_STDOUT_LOG"]) +def test_unparsable_stdout_log_limit_does_not_break_logging( + monkeypatch: pytest.MonkeyPatch, litellm_log: pytest.LogCaptureFixture, env_var: str +) -> None: + """ + Regression: the LiteLLM loggers' own filter reads these limits through get_env_int on every + record, so warning about a garbage value re-entered the filter and every log call raised + RecursionError. + """ + raw = f"not-a-number-{env_var}" + monkeypatch.setenv(env_var, raw) + + verbose_logger.warning("first line") + verbose_logger.warning("second line") + + messages = _logged_messages(litellm_log) + assert "first line" in messages + assert "second line" in messages + assert len([message for message in messages if raw in message]) == 1 diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_straiker.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_straiker.py index 292dd54105f..85886a2b9af 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_straiker.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_straiker.py @@ -1,5 +1,6 @@ import json from types import MappingProxyType, SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -1216,7 +1217,25 @@ def test_v3_initializer_reads_api_version_from_config(): assert g._webhook_url().endswith("/api/v3/detect") -@pytest.mark.parametrize("api_version", ["2024-09-01", "", "v2"]) +@pytest.mark.parametrize(("api_key", "expected"), [("c4ac433a-uuid", "v1"), (V3_KEY, "v3")]) +def test_blank_api_version_follows_key_prefix_without_warning( + api_key: str, expected: str, monkeypatch: pytest.MonkeyPatch +) -> None: + import litellm + from litellm._logging import verbose_proxy_logger + from litellm.types.guardrails import Guardrail, LitellmParams + + monkeypatch.setattr(litellm, "callbacks", litellm.callbacks.copy()) + params: Final = LitellmParams(guardrail="straiker", mode="pre_call", api_key=api_key, api_version="") + + with patch.object(verbose_proxy_logger, "warning") as warning: + g = initialize_guardrail(params, Guardrail(guardrail_name="straiker", litellm_params=params)) + + assert g.api_version == expected + warning.assert_not_called() + + +@pytest.mark.parametrize("api_version", ["2024-09-01", "v2"]) @pytest.mark.parametrize(("api_key", "expected"), [("c4ac433a-uuid", "v1"), (V3_KEY, "v3")]) def test_unknown_api_version_follows_key_prefix(api_version, api_key, expected, monkeypatch): import litellm diff --git a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py index 13049093d6c..4401e3e95c0 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py @@ -1670,6 +1670,31 @@ async def test_patch_guardrail_clears_scope_when_logging_only_mode_is_removed( handler.delete_in_memory_guardrail(stored_guardrail["guardrail_id"]) +@pytest.mark.asyncio +async def test_patch_guardrail_with_blank_value_clears_the_stored_value(mocker, monkeypatch, mock_guardrail_registry): + """ + Regression: blank litellm_params were dropped before validation, so a PATCH that cleared + api_base with "" returned 200 but kept the stored value. + """ + handler, stored_guardrail = _setup_patch_scope_guardrail( + mocker, + monkeypatch, + mock_guardrail_registry, + _PatchScopeSupportedGuardrail, + "patch_blank_value_test", + {"mode": "pre_call", "api_base": "https://old.example.com"}, + ) + request = PatchGuardrailRequest.model_validate({"litellm_params": {"api_base": ""}}) + + try: + await patch_guardrail(stored_guardrail["guardrail_id"], request, user_api_key_dict=MOCK_ADMIN_USER) + + persisted_guardrail = mock_guardrail_registry.update_guardrail_in_db.call_args.kwargs["guardrail"] + assert persisted_guardrail["litellm_params"].api_base is None + finally: + handler.delete_in_memory_guardrail(stored_guardrail["guardrail_id"]) + + @pytest.mark.asyncio async def test_patch_guardrail_tolerates_stored_unsupported_scope_on_unrelated_update( mocker, monkeypatch, mock_guardrail_registry, caplog diff --git a/tests/unit/proxy/guardrails/test_init_guardrails.py b/tests/unit/proxy/guardrails/test_init_guardrails.py index c8eeaf11fde..daf7ef1fd05 100644 --- a/tests/unit/proxy/guardrails/test_init_guardrails.py +++ b/tests/unit/proxy/guardrails/test_init_guardrails.py @@ -371,6 +371,44 @@ def test_init_guardrails_v2_skips_invalid_guardrail_instead_of_crashing_boot(): assert "healthy_presidio" in guardrail_names +def test_init_guardrails_v2_keeps_guardrail_whose_env_values_are_blank(): + """ + Regression: a config.yaml guardrail with `default_on: os.environ/GUARDRAIL_DEFAULT_ON` and + that variable exported blank (a Helm `value: ""`) resolved to "", pydantic rejected it on the + bool field, and the proxy logged the guardrail and booted without it. Blank now means unset, + so the guardrail boots with its defaults. + """ + from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER + + IN_MEMORY_GUARDRAIL_HANDLER.IN_MEMORY_GUARDRAILS.clear() + IN_MEMORY_GUARDRAIL_HANDLER.guardrail_id_to_custom_guardrail.clear() + + all_guardrails = [ + { + "guardrail_name": "presidio_with_blank_env_values", + "litellm_params": { + "guardrail": SupportedGuardrailIntegrations.PRESIDIO.value, + "mode": "pre_call", + "default_on": "", + "mask_request_content": "", + "presidio_analyzer_api_base": "https://fakelink.com/v1/presidio/analyze", + "presidio_anonymizer_api_base": "https://fakelink.com/v1/presidio/anonymize", + }, + }, + ] + + init_guardrails_v2(all_guardrails=all_guardrails) + + initialized = { + guardrail["guardrail_name"]: guardrail + for guardrail in IN_MEMORY_GUARDRAIL_HANDLER.IN_MEMORY_GUARDRAILS.values() + } + assert "presidio_with_blank_env_values" in initialized + litellm_params = initialized["presidio_with_blank_env_values"]["litellm_params"] + assert litellm_params.default_on is False + assert litellm_params.mask_request_content is None + + def test_init_guardrails_v2_stops_boot_when_a_custom_code_guardrail_does_not_compile(): from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER diff --git a/tests/unit/test_constants.py b/tests/unit/test_constants.py index 18ed6975605..0d0e77765b7 100644 --- a/tests/unit/test_constants.py +++ b/tests/unit/test_constants.py @@ -17,6 +17,19 @@ from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outco from unittest.mock import patch +_ENV_HELPERS = frozenset( + {"get_env_int", "get_env_float", "get_env_int_in_range", "get_env_int_or_none", "get_env_float_or_none"} +) + + +def _single_name_target(node: ast.stmt) -> str | None: + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + return node.target.id + if isinstance(node, ast.Assign) and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + return node.targets[0].id + return None + + def _build_constant_env_var_map() -> dict[str, str]: """ Build a mapping of CONSTANT_NAME -> ENV_VAR_NAME by parsing constants.py. @@ -29,13 +42,10 @@ def _build_constant_env_var_map() -> dict[str, str]: parsed = ast.parse(constants_source) for node in parsed.body: - if not isinstance(node, ast.Assign): + constant_name = _single_name_target(node) + if constant_name is None or not isinstance(node, (ast.Assign, ast.AnnAssign)) or node.value is None: continue - if len(node.targets) != 1 or not isinstance(node.targets[0], ast.Name): - continue - - constant_name = node.targets[0].id env_var_name = None for child in ast.walk(node.value): @@ -55,10 +65,10 @@ def _build_constant_env_var_map() -> dict[str, str]: env_var_name = child.args[0].value break - # get_env_int("ENV_NAME", default) + # get_env_int("ENV_NAME", default) and its siblings if ( isinstance(child.func, ast.Name) - and child.func.id == "get_env_int" + and child.func.id in _ENV_HELPERS and len(child.args) >= 1 and isinstance(child.args[0], ast.Constant) and isinstance(child.args[0].value, str) @@ -79,8 +89,10 @@ def _build_constant_env_var_map() -> dict[str, str]: (None, "48", 48), (None, None, 24), ("48", "72", 48), + (" ", None, 24), + ("", "48", 48), ], - ids=("canonical-only", "alias-only", "default", "canonical-wins"), + ids=("canonical-only", "alias-only", "default", "canonical-wins", "blank-canonical", "blank-canonical-alias-set"), ) def test_cli_jwt_expiration_hours_from_environment( monkeypatch: pytest.MonkeyPatch, @@ -105,12 +117,79 @@ def test_cli_jwt_expiration_hours_from_environment( importlib.reload(litellm.constants) +def _is_blank(value: object) -> bool: + return isinstance(value, str) and not value.strip() + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_constants_import_treats_blank_env_values_as_unset(blank: str) -> None: + """ + Regression: a numeric knob exported as a blank string (a Helm `value: ""` or an emptied .env + line) raised ValueError while importing litellm.constants, so the proxy crash-looped at boot. + Blank now means unset, so import succeeds with every env-backed constant blank, and every + numeric or optional constant keeps the value it has with the variable absent. + """ + constant_env_vars = _build_constant_env_var_map() + assert {"ROUTER_MAX_FALLBACKS", "LITELLM_MAX_STREAMING_DURATION_SECONDS"} <= constant_env_vars.keys() + env_vars = set(constant_env_vars.values()) + try: + with mock.patch.dict(os.environ): + for env_var in env_vars: + os.environ.pop(env_var, None) + importlib.reload(litellm.constants) + unset = {name: getattr(litellm.constants, name) for name in constant_env_vars} + with mock.patch.dict(os.environ, {env_var: blank for env_var in env_vars}): + importlib.reload(litellm.constants) + with_blank_env = {name: getattr(litellm.constants, name) for name in constant_env_vars} + finally: + importlib.reload(litellm.constants) + + numeric_or_optional = {name for name, value in unset.items() if value is None or type(value) in (int, float)} + changed = { + name: (unset[name], with_blank_env[name]) + for name in numeric_or_optional + if with_blank_env[name] != unset[name] and not _is_blank(with_blank_env[name]) + } + assert changed == {} + + +@pytest.mark.parametrize( + ("raw", "expected_timeout", "expected_explicit"), + [ + (None, constants.DEFAULT_REQUEST_TIMEOUT_SECONDS, False), + ("", constants.DEFAULT_REQUEST_TIMEOUT_SECONDS, False), + (" ", constants.DEFAULT_REQUEST_TIMEOUT_SECONDS, False), + ("fast", constants.DEFAULT_REQUEST_TIMEOUT_SECONDS, False), + (" 30 ", 30.0, True), + ], +) +def test_request_timeout_counts_as_explicit_only_when_it_parses( + monkeypatch: pytest.MonkeyPatch, raw: str | None, expected_timeout: float, expected_explicit: bool +) -> None: + """ + Regression: a blank REQUEST_TIMEOUT fell back to the 6000s sentinel but was still flagged as + explicitly set, so chat completion() kept the 6000s sentinel instead of its 600s fallback. + """ + if raw is None: + monkeypatch.delenv("REQUEST_TIMEOUT", raising=False) + else: + monkeypatch.setenv("REQUEST_TIMEOUT", raw) + try: + importlib.reload(litellm.constants) + assert litellm.constants.request_timeout == expected_timeout + assert litellm.constants.request_timeout_explicitly_set is expected_explicit + finally: + monkeypatch.undo() + importlib.reload(litellm.constants) + + @pytest.fixture() def _vcr_outcome_gate(request, vcr): install_live_call_probe(request, vcr) yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="session") def event_loop(): try: @@ -120,6 +199,7 @@ def event_loop(): yield loop loop.close() + @pytest.fixture(scope="function") def setup_and_teardown(event_loop): import litellm @@ -153,6 +233,7 @@ def setup_and_teardown(event_loop): if pending: event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "set_verbose": getattr(litellm, "set_verbose", False), @@ -176,17 +257,12 @@ def _populate_known_models_for_morph_tests(monkeypatch: pytest.MonkeyPatch) -> N monkeypatch.setattr( litellm, "models_by_provider", - { - provider: models.copy() - for provider, models in litellm.models_by_provider.items() - }, + {provider: models.copy() for provider, models in litellm.models_by_provider.items()}, ) litellm.add_known_models() -@pytest.mark.usefixtures( - "_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests" -) +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests") def test_morph_config_get_provider_info(): """Test that MorphChatConfig returns correct provider info.""" config = MorphChatConfig() @@ -207,9 +283,8 @@ def test_morph_config_get_provider_info(): assert api_base == "https://custom.morph.com" assert api_key == "key" -@pytest.mark.usefixtures( - "_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests" -) + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests") def test_morph_get_llm_provider(): """Test that get_llm_provider correctly identifies morph models.""" # Test with morph/model format @@ -219,9 +294,8 @@ def test_morph_get_llm_provider(): _, custom_llm_provider, _, _ = get_llm_provider("morph/morph-v3-fast") assert custom_llm_provider == "morph" -@pytest.mark.usefixtures( - "_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests" -) + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests") def test_morph_in_provider_lists(): """Test that morph is included in all necessary provider lists.""" import litellm @@ -242,9 +316,8 @@ def test_morph_in_provider_lists(): # Check models are in model_list after initialization assert all(model in litellm.model_list for model in ["morph/morph-v3-large", "morph/morph-v3-fast"]) -@pytest.mark.usefixtures( - "_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests" -) + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests") def test_morph_supported_params(): """Test that MorphChatConfig returns correct supported parameters.""" config = MorphChatConfig() @@ -258,9 +331,8 @@ def test_morph_supported_params(): assert all(param in supported_params for param in expected_params) -@pytest.mark.usefixtures( - "_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests" -) + +@pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown", "_populate_known_models_for_morph_tests") def test_morph_custom_llm_provider(): """Test that morph models are correctly identified.""" config = MorphChatConfig() diff --git a/tests/unit/types/test_guardrails.py b/tests/unit/types/test_guardrails.py index 8b74f7ce0fd..5c652a10e16 100644 --- a/tests/unit/types/test_guardrails.py +++ b/tests/unit/types/test_guardrails.py @@ -13,6 +13,7 @@ import litellm.litellm_core_utils import litellm.litellm_core_utils.litellm_logging from litellm import completion from litellm.integrations.custom_logger import CustomLogger +from litellm.types.guardrails import BaseLitellmParams, LitellmParams from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -157,3 +158,23 @@ def setup_and_teardown(): if hasattr(litellm, "in_memory_llm_clients_cache"): litellm.in_memory_llm_clients_cache.flush_cache() yield + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_litellm_params_treats_blank_values_as_unset(blank: str) -> None: + """ + Regression: `os.environ/X` with X exported blank resolves to "", which pydantic rejected on + bool fields (default_on, mask_request_content, fail_on_error), so the guardrail was dropped at + boot. Blank now means unset, so the field keeps the same default as when it is omitted. + """ + with_blank_values = LitellmParams( + guardrail="presidio", + mode="pre_call", + default_on=blank, + mask_request_content=blank, + api_key=blank, + ) + omitted = LitellmParams(guardrail="presidio", mode="pre_call") + assert with_blank_values.model_dump() == omitted.model_dump() + + assert BaseLitellmParams(fail_on_error=blank).model_dump() == BaseLitellmParams().model_dump()