diff --git a/litellm/_redis.py b/litellm/_redis.py index bb3a0974241..053514c0dbe 100644 --- a/litellm/_redis.py +++ b/litellm/_redis.py @@ -325,8 +325,19 @@ def _get_redis_client_logic(**env_overrides): value = get_secret(v) # type: ignore env_overrides[k] = value + environment_kwargs = _redis_kwargs_from_environment() + + # An explicitly configured connection target outranks REDIS_URL from the + # environment. Without this, the url branch below strips the caller's + # host/port/password and silently connects to whatever REDIS_URL names. + caller_named_a_target = any( + env_overrides.get(key) is not None for key in ("host", "startup_nodes", "sentinel_nodes") + ) + if caller_named_a_target and env_overrides.get("url") is None: + environment_kwargs.pop("url", None) + redis_kwargs = { - **_redis_kwargs_from_environment(), + **environment_kwargs, **env_overrides, } diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 43ddf302692..f441a1d3f84 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2151,6 +2151,41 @@ class PluginConfig(LiteLLMPydanticObjectBase): ) +class CoordinationRedisNode(LiteLLMPydanticObjectBase): + """A single startup node of a cluster-mode Redis used for proxy coordination.""" + + host: str = Field(description="hostname of the cluster node") + port: int = Field(description="port of the cluster node") + + +class CoordinationRedisParams(LiteLLMPydanticObjectBase): + """ + Connection params for the proxy's coordination Redis (cross-pod tpm/rpm rate + limits, spend tracking, pod lock manager, shared health checks), configured + independently of the response-cache backend in `litellm_settings.cache_params`. + """ + + model_config = ConfigDict(extra="allow", protected_namespaces=()) + + host: Optional[str] = Field(None, description="Redis hostname") + port: Optional[int] = Field(None, description="Redis port") + password: Optional[str] = Field(None, description="Redis password") + username: Optional[str] = Field(None, description="Redis username") + url: Optional[str] = Field(None, description="full Redis connection url, e.g. redis://:pass@host:6379") + ssl: Optional[bool] = Field(None, description="connect over TLS") + startup_nodes: Optional[List[CoordinationRedisNode]] = Field( + None, description="cluster-mode startup nodes; when set a cluster client is used" + ) + sentinel_nodes: Optional[List[List[Union[str, int]]]] = Field( + None, description="sentinel [host, port] pairs; when set a sentinel-managed client is used" + ) + sentinel_password: Optional[str] = Field(None, description="password for the sentinel nodes") + service_name: Optional[str] = Field(None, description="sentinel service name") + + def has_connection_target(self) -> bool: + return any(value is not None for value in (self.host, self.url, self.startup_nodes, self.sentinel_nodes)) + + class ConfigGeneralSettings(LiteLLMPydanticObjectBase): """ Documents all the fields supported by `general_settings` in config.yaml @@ -2166,6 +2201,15 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): use_google_kms: Optional[bool] = Field(None, description="decrypt keys with google kms") use_azure_key_vault: Optional[bool] = Field(None, description="load keys from azure key vault") master_key: Optional[str] = Field(None, description="require a key for all calls to proxy") + coordination_redis: Optional[CoordinationRedisParams] = Field( + None, + description=( + "standalone Redis for cross-pod coordination (tpm/rpm rate limits, " + "spend tracking, pod lock manager, shared health checks), configured " + "independently of the response-cache backend; takes precedence over " + "borrowing the `cache_params` Redis and over the REDIS_* env fallback" + ), + ) allow_cli_sso_verification_uri_complete: bool | None = Field( None, description="opt-in to RFC 8628 verification_uri_complete for the CLI SSO device flow, pre-filling the user_code in the browser. Off by default; intended for same-host clients where the device that starts the flow and the browser run on the same machine", diff --git a/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml b/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml index b353924f000..eb091cc72c5 100644 --- a/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml +++ b/litellm/proxy/example_config_yaml/multi_instance_simple_config.yaml @@ -5,8 +5,9 @@ model_list: api_key: my-fake-key api_base: os.environ/FAKE_OPENAI_API_BASE -litellm_settings: - cache: True - cache_params: - type: redis +general_settings: + coordination_redis: + host: os.environ/REDIS_HOST + port: os.environ/REDIS_PORT + password: os.environ/REDIS_PASSWORD diff --git a/litellm/proxy/example_config_yaml/spend_tracking_config.yaml b/litellm/proxy/example_config_yaml/spend_tracking_config.yaml index 60adadbd8d4..d66fd5fa601 100644 --- a/litellm/proxy/example_config_yaml/spend_tracking_config.yaml +++ b/litellm/proxy/example_config_yaml/spend_tracking_config.yaml @@ -7,9 +7,6 @@ model_list: general_settings: use_redis_transaction_buffer: true - -litellm_settings: - cache: True - cache_params: - type: redis - supported_call_types: [] \ No newline at end of file + coordination_redis: + host: os.environ/REDIS_HOST + port: os.environ/REDIS_PORT \ No newline at end of file diff --git a/litellm/proxy/management_endpoints/coordination_redis_endpoints.py b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py new file mode 100644 index 00000000000..7ab4e3019c3 --- /dev/null +++ b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py @@ -0,0 +1,431 @@ +""" +COORDINATION REDIS SETTINGS MANAGEMENT + +Endpoints for managing `general_settings.coordination_redis` - the standalone +Redis the proxy uses for cross-pod coordination (tpm/rpm rate limits, spend +tracking, pod lock manager, shared health checks), configured independently of +the response-cache backend. + +GET /coordination_redis/settings - Get the coordination Redis settings, field metadata, and which source is active +POST /coordination_redis/settings - Save coordination Redis settings to the database +POST /coordination_redis/settings/test - Test a coordination Redis connection with the provided credentials +""" + +import asyncio +import json +from collections.abc import Mapping +from contextlib import suppress +from datetime import datetime, timezone +from typing import Optional + +from fastapi import APIRouter, Depends, Header, HTTPException +from pydantic import BaseModel, Field, TypeAdapter, ValidationError + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm._uuid import uuid +from litellm.caching.caching import RedisCache +from litellm.caching.redis_cluster_cache import RedisClusterCache +from litellm.proxy._types import ( + AUDIT_ACTIONS, + CoordinationRedisParams, + LiteLLM_AuditLogs, + LitellmTableNames, + LitellmUserRoles, + UserAPIKeyAuth, +) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.utils import invalidate_config_param +from litellm.repositories.config_repository import ConfigRepository +from litellm.secret_managers.main import get_secret_str +from litellm.types.management_endpoints import ( + COORDINATION_REDIS_SETTINGS_FIELDS, + CoordinationRedisSettingsField, + CoordinationRedisSource, +) + +router = APIRouter() + +_GENERAL_SETTINGS_PARAM_NAME = "general_settings" +_COORDINATION_REDIS_KEY = "coordination_redis" + +# Fields that carry credentials. Redacted on read so a plaintext Redis / +# Sentinel password never leaves the server, and scrubbed out of connection-test +# error strings. `url` is here because a Redis url can embed a password inline +# (e.g. redis://:secret@host:6379/1). +_SENSITIVE_FIELDS: frozenset[str] = frozenset({"password", "sentinel_password", "url"}) + +_REDACTED_VALUE = "***REDACTED***" + +_ENV_REF_PREFIX = "os.environ/" + +_PING_TIMEOUT_SECONDS = 5.0 + +_SETTINGS_ADAPTER: TypeAdapter[dict[str, object]] = TypeAdapter(dict[str, object]) + + +def _enforce_proxy_admin(user_api_key_dict: UserAPIKeyAuth) -> None: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException( + status_code=403, + detail={"error": "Only proxy admins can manage coordination Redis settings"}, + ) + + +def _resolve_env_ref(value: object) -> object: + """Resolve an `os.environ/VAR` reference to its value, passing anything else through.""" + if isinstance(value, str) and value.startswith(_ENV_REF_PREFIX): + return get_secret_str(value) + return value + + +def _resolve_env_refs(settings: Mapping[str, object]) -> dict[str, object]: + return {key: _resolve_env_ref(value) for key, value in settings.items()} + + +def _redact_credentials(settings: Mapping[str, object]) -> dict[str, object]: + """Replace credential-bearing values with a fixed marker, keeping the rest intact.""" + return { + key: (_REDACTED_VALUE if key in _SENSITIVE_FIELDS and value is not None else value) + for key, value in settings.items() + } + + +def _redact_all_values(settings: Optional[Mapping[str, object]]) -> dict[str, object]: + """Replace every value with a fixed marker, preserving the key set. + + The audit row shows *which* fields changed without the audit table becoming + a credential-harvest sink. + """ + if not settings: + return {} + return {key: _REDACTED_VALUE for key in settings} + + +def _credential_values(settings: Mapping[str, object]) -> tuple[str, ...]: + return tuple( + str(value) for key, value in settings.items() if key in _SENSITIVE_FIELDS and isinstance(value, (str, int)) + ) + + +def _scrub_credentials(message: str, settings: Mapping[str, object]) -> str: + """Strip any credential value the caller supplied out of an error string. + + Redis client errors routinely echo the connection url (password inline) or + the auth error back to the caller. + """ + scrubbed = message + for secret in _credential_values(settings): + if secret: + scrubbed = scrubbed.replace(secret, _REDACTED_VALUE) + return scrubbed + + +def _merge_over_saved( + incoming: Mapping[str, object], + saved: Mapping[str, object], +) -> dict[str, object]: + """Restore the real credential behind every value the caller echoed back redacted. + + GET returns credentials as ``***REDACTED***``; an admin who edits the + non-secret fields and re-submits would otherwise test (and save) the marker + as the password. + """ + return { + key: (saved[key] if value == _REDACTED_VALUE and key in saved else value) for key, value in incoming.items() + } + + +def _validated_params(settings: Mapping[str, object]) -> CoordinationRedisParams: + """Validate settings the way startup does: resolve env refs, then require a connection target.""" + try: + params = CoordinationRedisParams(**_resolve_env_refs(settings)) + except ValidationError as e: + invalid_fields = sorted({str(error["loc"][0]) for error in e.errors() if error["loc"]}) + raise HTTPException( + status_code=400, + detail={"error": f"Invalid coordination_redis settings for fields: {invalid_fields}"}, + ) + if not params.has_connection_target(): + raise HTTPException( + status_code=400, + detail={ + "error": ( + "coordination_redis needs a connection target: " + "set one of host, url, startup_nodes, or sentinel_nodes" + ) + }, + ) + return params + + +async def _read_general_settings() -> dict[str, object]: + """Read the persisted `general_settings` config row (empty when unset or no DB).""" + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + return {} + config_param = await ConfigRepository(prisma_client).get_param(_GENERAL_SETTINGS_PARAM_NAME) + if config_param is None or config_param.param_value is None: + return {} + return _SETTINGS_ADAPTER.validate_python(config_param.param_value) + + +async def get_persisted_coordination_redis_settings() -> Optional[dict[str, object]]: + """The coordination_redis block saved to the database, if any. + + Read at startup so settings saved from the admin UI take effect on the next + boot, and used here so a read reports what the proxy would boot with. + """ + persisted = (await _read_general_settings()).get(_COORDINATION_REDIS_KEY) + if isinstance(persisted, dict): + return _SETTINGS_ADAPTER.validate_python(persisted) + return None + + +async def _current_coordination_redis_settings() -> Optional[dict[str, object]]: + """The coordination_redis block the proxy would boot with. + + The persisted row wins over the yaml-loaded config state because startup + applies the DB `general_settings` row over the file config. + """ + from litellm.proxy.proxy_server import proxy_config + + persisted = await get_persisted_coordination_redis_settings() + if persisted is not None: + return persisted + + config_state = _SETTINGS_ADAPTER.validate_python(proxy_config.get_config_state()) + general_settings = config_state.get(_GENERAL_SETTINGS_PARAM_NAME) + if not isinstance(general_settings, dict): + return None + from_file = general_settings.get(_COORDINATION_REDIS_KEY) + if isinstance(from_file, dict): + return _SETTINGS_ADAPTER.validate_python(from_file) + return None + + +def _coordination_redis_source(settings: Optional[Mapping[str, object]]) -> Optional[CoordinationRedisSource]: + """Which source the proxy's coordination Redis comes from, in startup precedence order. + + Mirrors `ProxyConfig._init_coordination_redis` -> `ProxyConfig._init_cache`: + an explicit block wins, else a plain-Redis response-cache backend is + borrowed, else the REDIS_* environment fallback applies. + """ + from litellm.proxy.proxy_server import _environment_has_redis_connection_target + + if settings: + return "coordination_redis" + cache_backend = litellm.cache.cache if litellm.cache is not None else None + if isinstance(cache_backend, (RedisCache, RedisClusterCache)): + return "cache_backend" + if _environment_has_redis_connection_target(): + return "environment" + return None + + +def _log_audit_task_exception(task: "asyncio.Task[None]") -> None: + """Surface a fire-and-forget audit-log task failure as a warning.""" + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + verbose_proxy_logger.warning("Failed to write coordination-redis-settings audit log: %s", exc) + + +async def _emit_coordination_redis_audit_log( + *, + action: AUDIT_ACTIONS, + before_settings: Optional[Mapping[str, object]], + after_settings: Optional[Mapping[str, object]], + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: Optional[str], +) -> None: + """Emit an audit-log row for a /coordination_redis/settings mutation.""" + if litellm.store_audit_logs is not True: + return + + from litellm.proxy.management_helpers.audit_logs import create_audit_log_for_update + from litellm.proxy.proxy_server import litellm_proxy_admin_name + + task = asyncio.create_task( + create_audit_log_for_update( + request_data=LiteLLM_AuditLogs( + id=str(uuid.uuid4()), + updated_at=datetime.now(timezone.utc), + changed_by=litellm_changed_by or user_api_key_dict.user_id or litellm_proxy_admin_name, + changed_by_api_key=user_api_key_dict.api_key, + table_name=LitellmTableNames.CONFIG_TABLE_NAME, + object_id=_COORDINATION_REDIS_KEY, + action=action, + updated_values=json.dumps({"settings": _redact_all_values(after_settings)}, default=str), + before_value=json.dumps({"settings": _redact_all_values(before_settings)}, default=str), + ) + ) + ) + task.add_done_callback(_log_audit_task_exception) + + +class CoordinationRedisSettingsResponse(BaseModel): + values: dict[str, object] = Field(description="Current coordination Redis settings, with credentials redacted") + fields: list[CoordinationRedisSettingsField] = Field( + description="List of all configurable coordination Redis settings with metadata" + ) + source: Optional[CoordinationRedisSource] = Field( + description="Where the proxy's coordination Redis comes from; null when it has none" + ) + + +class CoordinationRedisSettingsRequest(BaseModel): + settings: dict[str, object] = Field(description="Coordination Redis connection params") + + +class CoordinationRedisTestResponse(BaseModel): + status: str = Field(description="Connection status: 'healthy' or 'unhealthy'") + error: Optional[str] = Field(default=None, description="Error message if the connection failed") + + +@router.get( + "/coordination_redis/settings", + tags=["Coordination Redis Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=CoordinationRedisSettingsResponse, +) +async def get_coordination_redis_settings( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> CoordinationRedisSettingsResponse: + """ + Get the coordination Redis configuration and available settings. + + Returns: + - values: current coordination Redis settings, with password/sentinel_password/url redacted + - fields: all configurable settings with their metadata (type, description, default, section) + - source: "coordination_redis" | "cache_backend" | "environment" | null + """ + _enforce_proxy_admin(user_api_key_dict) + + settings = await _current_coordination_redis_settings() + source = _coordination_redis_source(settings) + + values = _redact_credentials(settings or {}) + fields = [field.model_copy(deep=True) for field in COORDINATION_REDIS_SETTINGS_FIELDS] + for field in fields: + if field.field_name in values: + field.field_value = values[field.field_name] + + return CoordinationRedisSettingsResponse(values=values, fields=fields, source=source) + + +@router.post( + "/coordination_redis/settings", + tags=["Coordination Redis Settings"], + dependencies=[Depends(user_api_key_auth)], +) +async def update_coordination_redis_settings( + request: CoordinationRedisSettingsRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + litellm_changed_by: Optional[str] = Header( + None, + description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability", + ), +) -> dict[str, object]: + """ + Save coordination Redis settings under `general_settings.coordination_redis`. + + Parameters: + - settings: dict - Redis connection params (host, port, username, password, url, ssl, startup_nodes, sentinel_nodes, sentinel_password, service_name). Values may be `os.environ/VAR` references, which are stored as written and resolved at startup + + The settings are written to the `general_settings` row of LiteLLM_Config, + which startup merges over the yaml config; the proxy picks them up on its + next restart. + """ + from litellm.proxy.proxy_server import prisma_client, store_model_in_db + + _enforce_proxy_admin(user_api_key_dict) + + if prisma_client is None: + raise HTTPException( + status_code=500, + detail={"error": "Database not connected. Please connect a database."}, + ) + + if store_model_in_db is not True: + raise HTTPException( + status_code=500, + detail={"error": "Set `'STORE_MODEL_IN_DB='True'` in your env to enable this feature."}, + ) + + saved_settings = await _current_coordination_redis_settings() + settings = _merge_over_saved(request.settings, saved_settings or {}) + _validated_params(settings) + + general_settings = await _read_general_settings() + before_settings = general_settings.get(_COORDINATION_REDIS_KEY) + action: AUDIT_ACTIONS = "updated" if isinstance(before_settings, dict) else "created" + + await ConfigRepository(prisma_client).set_param( + param_name=_GENERAL_SETTINGS_PARAM_NAME, + param_value={**general_settings, _COORDINATION_REDIS_KEY: settings}, + ) + await invalidate_config_param(_GENERAL_SETTINGS_PARAM_NAME) + + # coordination_redis carries Redis credentials and decides where cross-pod + # rate-limit and spend state lives; an admin repointing it is a + # data-routing pivot, so make the change traceable. + await _emit_coordination_redis_audit_log( + action=action, + before_settings=before_settings if isinstance(before_settings, dict) else None, + after_settings=settings, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + + return { + "message": "Coordination Redis settings updated successfully. Restart the proxy to apply them.", + "status": "success", + "settings": _redact_credentials(settings), + } + + +@router.post( + "/coordination_redis/settings/test", + tags=["Coordination Redis Settings"], + dependencies=[Depends(user_api_key_auth)], + response_model=CoordinationRedisTestResponse, +) +async def check_coordination_redis_connection( + request: CoordinationRedisSettingsRequest, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> CoordinationRedisTestResponse: + """ + Test a coordination Redis connection with the provided credentials. + + Parameters: + - settings: dict - Redis connection params to test. Credential fields sent back as `***REDACTED***` fall back to the saved value + + Builds a throwaway client (never touching global state) and pings it. + """ + from litellm.proxy.proxy_server import _build_redis_usage_cache + + _enforce_proxy_admin(user_api_key_dict) + + saved_settings = await _current_coordination_redis_settings() + settings = _merge_over_saved(request.settings, saved_settings or {}) + params = _validated_params(settings) + + redis_cache: Optional[RedisCache] = None + try: + redis_cache = _build_redis_usage_cache(params.model_dump(exclude_none=True)) + await asyncio.wait_for(redis_cache.ping(), timeout=_PING_TIMEOUT_SECONDS) + return CoordinationRedisTestResponse(status="healthy") + except asyncio.TimeoutError: + return CoordinationRedisTestResponse( + status="unhealthy", + error=f"Connection timed out after {_PING_TIMEOUT_SECONDS}s", + ) + except Exception as e: # noqa: BLE001 # any client/connection failure is a health verdict, not a 500 + return CoordinationRedisTestResponse(status="unhealthy", error=_scrub_credentials(str(e), settings)) + finally: + if redis_cache is not None: + with suppress(Exception): + await redis_cache.disconnect() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 38f944b897c..f0ca1f6396f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -75,6 +75,7 @@ from litellm.proxy._types import ( ConfigGeneralSettings, ConfigList, ConfigYAML, + CoordinationRedisParams, EnterpriseLicenseData, FieldDetail, InvitationClaim, @@ -362,6 +363,10 @@ from litellm.proxy.management_endpoints.cache_settings_endpoints import ( from litellm.proxy.management_endpoints.callback_management_endpoints import ( router as callback_management_endpoints_router, ) +from litellm.proxy.management_endpoints.coordination_redis_endpoints import ( + get_persisted_coordination_redis_settings, + router as coordination_redis_settings_router, +) from litellm.proxy.management_endpoints.common_utils import ( _user_has_admin_privileges, _user_has_admin_view, @@ -926,6 +931,16 @@ async def proxy_startup_event(app: FastAPI): asyncio.create_task(_run_pw_migration()) + ## A coordination_redis block saved from the admin UI lives in the database, + ## which is only reachable once the prisma client exists. Apply it here, before + ## the coordination Redis is published to its consumers below. + db_coordination_redis_cache = await ProxyStartupEvent._init_coordination_redis_from_db( + litellm_settings=proxy_config.get_config_state().get("litellm_settings") or {}, + llm_router=llm_router, + ) + if db_coordination_redis_cache is not None: + _set_redis_usage_cache(db_coordination_redis_cache) + ## use_redis_transaction_buffer: fall back to a standalone Redis (REDIS_* env) ## when the proxy cache backend is not Redis ## transaction_buffer_redis_cache = redis_usage_cache @@ -3550,20 +3565,99 @@ def _apply_ssrf_general_settings(settings: Mapping[str, object]) -> None: ) +def _set_redis_usage_cache(coordination_redis_cache: RedisCache | None) -> None: + """Publish the resolved coordination Redis to the consumers that read it directly.""" + global redis_usage_cache + redis_usage_cache = coordination_redis_cache + + +def _resolve_coordination_redis_env_refs(raw_params: Mapping[str, object]) -> dict[str, object]: + """Resolve `os.environ/VAR` references in a coordination_redis block.""" + return { + key: (get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value) + for key, value in raw_params.items() + } + + +def _build_redis_usage_cache(redis_params: Mapping[str, object]) -> RedisCache: + """ + Builds the proxy's coordination Redis client from resolved connection + params. Cluster-mode targets (explicit `startup_nodes` or the + REDIS_CLUSTER_NODES env var) get a `RedisClusterCache`, so consumers that + branch on cluster mode (e.g. the v3 rate limiter) take the cluster path; + everything else (host/url/sentinel) gets a plain `RedisCache`. + """ + startup_nodes = redis_params.get("startup_nodes") + if startup_nodes is None: + env_cluster_nodes = get_secret_str("REDIS_CLUSTER_NODES") + if env_cluster_nodes is not None: + startup_nodes = json.loads(env_cluster_nodes) + non_node_params = {key: value for key, value in redis_params.items() if key != "startup_nodes"} + if startup_nodes: + return RedisClusterCache(startup_nodes=startup_nodes, **non_node_params) + return RedisCache(**non_node_params) + + +def _environment_has_redis_connection_target() -> bool: + """ + Whether the REDIS_* environment variables name a Redis to connect to (host, + url, cluster nodes, or sentinel nodes). Read-only: callers that only need to + know whether the env fallback would apply use this instead of building a + client. + """ + redis_env_kwargs = litellm._redis._redis_kwargs_from_environment() + return ( + "host" in redis_env_kwargs + or "url" in redis_env_kwargs + or get_secret_str("REDIS_CLUSTER_NODES") is not None + or get_secret_str("REDIS_SENTINEL_NODES") is not None + ) + + def _build_redis_usage_cache_from_environment() -> RedisCache | None: """ - Builds a standalone RedisCache from REDIS_* environment variables. + Builds a standalone coordination Redis from REDIS_* environment variables. Lets the proxy's coordination Redis (cross-pod tpm/rpm rate limits, spend tracking, pod lock manager) run when the response-cache backend is not a plain Redis KV cache (e.g. a semantic cache, disk, or s3). - Returns None when no Redis host or url is set in the environment. + Returns None when the environment carries no connection target (host, url, + cluster nodes, or sentinel nodes). """ - redis_env_kwargs = litellm._redis._redis_kwargs_from_environment() - if "host" not in redis_env_kwargs and "url" not in redis_env_kwargs: + if not _environment_has_redis_connection_target(): return None - return RedisCache(**redis_env_kwargs) + return _build_redis_usage_cache(litellm._redis._redis_kwargs_from_environment()) + + +def _attach_redis_usage_cache(redis_cache: RedisCache, enable_redis_auth_cache: bool) -> None: + """ + Wires an established coordination Redis into the proxy-level caches that + consume it directly: the spend counter cache, the cluster-wide config + cache, and (only when opted in) the virtual-key auth cache. + """ + spend_counter_cache.attach_redis_cache( + redis_cache, + default_redis_ttl=litellm.default_redis_ttl, + ) + if enable_redis_auth_cache is True: + user_api_key_cache.attach_redis_cache( + redis_cache, + default_redis_ttl=litellm.default_redis_ttl, + ) + verbose_proxy_logger.info( + "enable_redis_auth_cache=True: attached Redis to " + "user_api_key_cache — virtual-key lookups are now " + "shared across all proxy workers." + ) + else: + verbose_proxy_logger.info( + "enable_redis_auth_cache is not set: user_api_key_cache " + "remains in-memory only (per-worker). Set " + "litellm_settings.enable_redis_auth_cache: true to share " + "the auth cache across workers and reduce DB load." + ) + litellm_config_cache.redis_cache = redis_cache class ProxyConfig: @@ -3764,12 +3858,52 @@ class ProxyConfig: team_config = self._get_team_config(team_id=team_id, all_teams_config=all_teams_config) return team_config + def _init_coordination_redis(self, config: dict) -> RedisCache | None: + """ + Builds the coordination Redis from `general_settings.coordination_redis` + when present, attaching it to the proxy-level caches. Runs before cache + init, so an explicit block takes precedence over borrowing the + response-cache Redis and over the REDIS_* env fallback. Returns the + built client (None when the block is absent) for the caller to publish. + """ + settings = config.get("general_settings") or {} + litellm_settings = config.get("litellm_settings") or {} + raw_params = settings.get("coordination_redis") + if raw_params is None: + return None + if not isinstance(raw_params, dict): + raise ValueError("general_settings.coordination_redis must be a mapping of Redis connection params") + + coordination_params = CoordinationRedisParams(**_resolve_coordination_redis_env_refs(raw_params)) + if not coordination_params.has_connection_target(): + raise ValueError( + "general_settings.coordination_redis needs a connection target: " + "set one of host, url, startup_nodes, or sentinel_nodes" + ) + + coordination_redis_cache = _build_redis_usage_cache(coordination_params.model_dump(exclude_none=True)) + _attach_redis_usage_cache( + coordination_redis_cache, + enable_redis_auth_cache=litellm_settings.get("enable_redis_auth_cache", False) is True, + ) + verbose_proxy_logger.info( + "coordination_redis: using a standalone Redis from general_settings " + "for usage tracking, rate limiting, and cross-pod coordination." + ) + return coordination_redis_cache + def _init_cache( self, cache_params: dict, enable_redis_auth_cache: bool = False, - ): - global redis_usage_cache, llm_router, general_settings + ) -> RedisCache | None: + """ + Initializes the response cache and resolves the coordination Redis. + + Returns the coordination Redis for the caller to publish: an explicit + coordination_redis block already set wins, else a plain-Redis response + cache backend is borrowed, else the REDIS_* environment fallback applies. + """ from litellm import Cache if "default_in_memory_ttl" in cache_params: @@ -3780,49 +3914,29 @@ class ProxyConfig: litellm.cache = Cache(**cache_params) + resolved_usage_cache = redis_usage_cache cache_backend = litellm.cache.cache if litellm.cache is not None else None - if isinstance(cache_backend, (RedisCache, RedisClusterCache)): - ## INIT PROXY REDIS USAGE CLIENT ## - redis_usage_cache = cache_backend - elif redis_usage_cache is None: - redis_usage_cache = _build_redis_usage_cache_from_environment() - if redis_usage_cache is not None: - verbose_proxy_logger.info( - "Cache backend %s is not a Redis KV cache; built a standalone " - "Redis from REDIS_* environment variables for usage tracking, " - "rate limiting, and cross-pod coordination.", - type(cache_backend).__name__, - ) - - if redis_usage_cache is not None: - spend_counter_cache.attach_redis_cache( - redis_usage_cache, - default_redis_ttl=litellm.default_redis_ttl, - ) - # Note: PKCE verifier storage uses redis_usage_cache directly (not - # user_api_key_cache) to avoid routing all API-key lookups through Redis. - if enable_redis_auth_cache is True: - user_api_key_cache.attach_redis_cache( - redis_usage_cache, - default_redis_ttl=litellm.default_redis_ttl, - ) - verbose_proxy_logger.info( - "enable_redis_auth_cache=True: attached Redis to " - "user_api_key_cache — virtual-key lookups are now " - "shared across all proxy workers." - ) + if resolved_usage_cache is None: + if isinstance(cache_backend, (RedisCache, RedisClusterCache)): + ## INIT PROXY REDIS USAGE CLIENT ## + resolved_usage_cache = cache_backend else: - verbose_proxy_logger.info( - "enable_redis_auth_cache is not set: user_api_key_cache " - "remains in-memory only (per-worker). Set " - "litellm_settings.enable_redis_auth_cache: true to share " - "the auth cache across workers and reduce DB load." - ) - litellm_config_cache.redis_cache = redis_usage_cache + resolved_usage_cache = _build_redis_usage_cache_from_environment() + if resolved_usage_cache is not None: + verbose_proxy_logger.info( + "Cache backend %s is not a Redis KV cache; built a standalone " + "Redis from REDIS_* environment variables for usage tracking, " + "rate limiting, and cross-pod coordination.", + type(cache_backend).__name__, + ) + + if resolved_usage_cache is not None: # Note: PKCE verifier storage uses redis_usage_cache directly (not # user_api_key_cache) to avoid routing all API-key lookups through Redis. + _attach_redis_usage_cache(resolved_usage_cache, enable_redis_auth_cache) elif litellm_config_cache.redis_cache is None: verbose_proxy_logger.info("litellm_config_cache: no Redis configured; cluster-wide cache sharing disabled.") + return resolved_usage_cache def switch_on_llm_response_caching(self): """ @@ -4067,6 +4181,11 @@ class ProxyConfig: self._load_environment_variables(config=config) + ## Coordination Redis (before cache init, so the explicit block wins) + coordination_redis_cache = self._init_coordination_redis(config=config) + if coordination_redis_cache is not None: + _set_redis_usage_cache(coordination_redis_cache) + ## Callback settings callback_settings = config.get("callback_settings", {}) if callback_settings: @@ -4147,9 +4266,11 @@ class ProxyConfig: cache_params[key] = get_secret(value) ## to pass a complete url, or set ssl=True, etc. just set it as `os.environ[REDIS_URL] = `, _redis.py checks for REDIS specific environment variables - self._init_cache( - cache_params=cache_params, - enable_redis_auth_cache=litellm_settings.get("enable_redis_auth_cache", False) is True, + _set_redis_usage_cache( + self._init_cache( + cache_params=cache_params, + enable_redis_auth_cache=litellm_settings.get("enable_redis_auth_cache", False) is True, + ) ) if litellm.cache is not None: verbose_proxy_logger.debug(f"{blue_color_code}Set Cache on LiteLLM Proxy{reset_color_code}") @@ -7294,6 +7415,46 @@ class ProxyStartupEvent: "Redis for the transaction buffer." ) + @staticmethod + async def _init_coordination_redis_from_db( + litellm_settings: Mapping[str, object], + llm_router: Optional[Router], + ) -> RedisCache | None: + """ + Applies a coordination_redis block saved to the database, which the admin + UI writes and the config file therefore never carries. + + Returns None when nothing is persisted or the persisted block names no + connection target, leaving the file/env resolution untouched. + """ + try: + persisted = await get_persisted_coordination_redis_settings() + except Exception as e: # noqa: BLE001 # a config-row read failure must not block proxy startup + verbose_proxy_logger.warning("Could not read coordination_redis from the database: %s", e) + return None + if persisted is None: + return None + + coordination_params = CoordinationRedisParams(**_resolve_coordination_redis_env_refs(persisted)) + if not coordination_params.has_connection_target(): + verbose_proxy_logger.warning( + "coordination_redis saved in the database names no connection target; ignoring it." + ) + return None + + coordination_redis_cache = _build_redis_usage_cache(coordination_params.model_dump(exclude_none=True)) + _attach_redis_usage_cache( + coordination_redis_cache, + enable_redis_auth_cache=litellm_settings.get("enable_redis_auth_cache", False) is True, + ) + if llm_router is not None and llm_router.cache.redis_cache is None: + llm_router._update_redis_cache(cache=coordination_redis_cache) + verbose_proxy_logger.info( + "coordination_redis: using the standalone Redis saved in the database " + "for usage tracking, rate limiting, and cross-pod coordination." + ) + return coordination_redis_cache + @staticmethod def _get_transaction_buffer_redis_cache( general_settings: dict, @@ -15835,6 +15996,7 @@ app.include_router(cost_tracking_settings_router) app.include_router(router_settings_router) app.include_router(fallback_management_router) app.include_router(cache_settings_router) +app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(enterprise_router) app.include_router(ui_discovery_endpoints_router) diff --git a/litellm/types/management_endpoints/__init__.py b/litellm/types/management_endpoints/__init__.py index 5c5bcb2e754..3b501443edd 100644 --- a/litellm/types/management_endpoints/__init__.py +++ b/litellm/types/management_endpoints/__init__.py @@ -7,6 +7,12 @@ from .cache_settings_endpoints import ( REDIS_TYPE_DESCRIPTIONS, CacheSettingsField, ) +from .coordination_redis_endpoints import ( + COORDINATION_REDIS_SETTINGS_FIELDS, + CoordinationRedisSection, + CoordinationRedisSettingsField, + CoordinationRedisSource, +) from .router_settings_endpoints import ( ROUTER_SETTINGS_FIELDS, ROUTING_STRATEGY_DESCRIPTIONS, @@ -20,4 +26,8 @@ __all__ = [ "CACHE_SETTINGS_FIELDS", "REDIS_TYPE_DESCRIPTIONS", "CacheSettingsField", + "COORDINATION_REDIS_SETTINGS_FIELDS", + "CoordinationRedisSection", + "CoordinationRedisSettingsField", + "CoordinationRedisSource", ] diff --git a/litellm/types/management_endpoints/coordination_redis_endpoints.py b/litellm/types/management_endpoints/coordination_redis_endpoints.py new file mode 100644 index 00000000000..b6889d83323 --- /dev/null +++ b/litellm/types/management_endpoints/coordination_redis_endpoints.py @@ -0,0 +1,105 @@ +""" +Types and field definitions for coordination Redis settings management endpoints +""" + +from typing import Literal, Optional + +from pydantic import BaseModel + +CoordinationRedisSection = Literal["connection", "cluster", "sentinel"] + +CoordinationRedisSource = Literal["coordination_redis", "cache_backend", "environment"] + + +class CoordinationRedisSettingsField(BaseModel): + field_name: str + field_type: str + field_value: Optional[object] = None + field_description: str + field_default: Optional[object] = None + ui_field_name: str + section: CoordinationRedisSection + + +COORDINATION_REDIS_SETTINGS_FIELDS: list[CoordinationRedisSettingsField] = [ + CoordinationRedisSettingsField( + field_name="host", + field_type="String", + field_description="Redis server hostname or IP address", + ui_field_name="Host", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="port", + field_type="Integer", + field_description="Redis server port number", + field_default=6379, + ui_field_name="Port", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="username", + field_type="String", + field_description="Redis server username (if required)", + ui_field_name="Username", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="password", + field_type="String", + field_description="Redis server password", + ui_field_name="Password", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="url", + field_type="String", + field_description=( + "Full Redis connection URL (e.g. redis://:password@host:6379/1). " + "Set this instead of the discrete host/port/username/password fields." + ), + ui_field_name="Redis URL", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="ssl", + field_type="Boolean", + field_description="Connect to Redis over TLS", + field_default=False, + ui_field_name="SSL", + section="connection", + ), + CoordinationRedisSettingsField( + field_name="startup_nodes", + field_type="List", + field_description=( + "Cluster-mode startup nodes (e.g. [{'host': '127.0.0.1', 'port': 7001}]). " + "When set, a Redis Cluster client is used." + ), + ui_field_name="Cluster Startup Nodes", + section="cluster", + ), + CoordinationRedisSettingsField( + field_name="sentinel_nodes", + field_type="List", + field_description=( + "Sentinel [host, port] pairs (e.g. [['localhost', 26379]]). When set, a Sentinel-managed client is used." + ), + ui_field_name="Sentinel Nodes", + section="sentinel", + ), + CoordinationRedisSettingsField( + field_name="sentinel_password", + field_type="String", + field_description="Password for the Redis Sentinel nodes", + ui_field_name="Sentinel Password", + section="sentinel", + ), + CoordinationRedisSettingsField( + field_name="service_name", + field_type="String", + field_description="Master service name for Redis Sentinel", + ui_field_name="Service Name", + section="sentinel", + ), +] diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index 6feffe036bd..1a51dd0d0a7 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -189,7 +189,7 @@ litellm_settings: langfuse_host: https://us.cloud.langfuse.com # cache: true # [OPTIONAL] use for caching responses # enable_caching_on_provider_specific_optional_params: True # Include provider-specific params in cache keys - # cache_params: # And for shared health check + # cache_params: # type: redis # host: localhost # port: 6379 @@ -228,8 +228,11 @@ general_settings: proxy_batch_write_at: 1 database_connection_pool_limit: 10 # background_health_checks: true - # use_shared_health_check: true + # use_shared_health_check: true # needs a coordination Redis (below) # health_check_interval: 30 + # coordination_redis: # standalone Redis for cross-pod coordination: rate limits, spend tracking, pod locks, shared health checks + # host: localhost + # port: 6379 # cancel_on_disconnect: true # cancel the in-flight upstream LLM request (non-streaming) when the client disconnects, freeing backend capacity (e.g. a vLLM GPU slot) # database_url: "postgresql://:@:/" # [OPTIONAL] use for token-based auth to proxy diff --git a/tests/test_litellm/proxy/management_endpoints/test_coordination_redis_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_coordination_redis_endpoints.py new file mode 100644 index 00000000000..4e6bfc4c063 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/test_coordination_redis_endpoints.py @@ -0,0 +1,577 @@ +""" +Unit tests for coordination Redis settings management endpoints +""" + +import asyncio +import json +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi import HTTPException + +sys.path.insert(0, os.path.abspath("../../../..")) # Adds the parent directory to the system path + +import litellm +from litellm.caching.caching import RedisCache +from litellm.caching.redis_cluster_cache import RedisClusterCache +from litellm.proxy._types import LitellmTableNames, LitellmUserRoles +from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.proxy.management_endpoints.coordination_redis_endpoints import ( + _REDACTED_VALUE, + CoordinationRedisSettingsRequest, + get_coordination_redis_settings, + check_coordination_redis_connection, + update_coordination_redis_settings, +) +from litellm.types.management_endpoints.coordination_redis_endpoints import ( + COORDINATION_REDIS_SETTINGS_FIELDS, +) + +_SAVED_SETTINGS = { + "host": "coord-redis.example.com", + "port": 6379, + "password": "super-secret-redis-pw", + "url": "redis://:super-secret-redis-pw@coord-redis.example.com:6379", + "sentinel_password": "super-secret-sentinel-pw", +} + + +def _admin_auth() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="hashed", + user_id="admin-user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + +def _prisma_with_general_settings(general_settings: dict | None) -> MagicMock: + """A prisma client whose LiteLLM_Config `general_settings` row holds ``general_settings``.""" + row = None + if general_settings is not None: + row = MagicMock() + row.param_value = json.dumps(general_settings) + + mock_prisma = MagicMock() + mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=row) + mock_prisma.db.litellm_config.upsert = AsyncMock() + return mock_prisma + + +def _proxy_config(file_general_settings: dict | None = None) -> MagicMock: + proxy_config = MagicMock() + proxy_config.get_config_state = MagicMock( + return_value={"general_settings": file_general_settings or {}}, + ) + return proxy_config + + +# ── GET /coordination_redis/settings ────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_get_redacts_every_credential_field(): + """password, sentinel_password and the (password-bearing) url never leave the + server in plaintext; non-credential fields come back untouched.""" + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + _prisma_with_general_settings({"coordination_redis": _SAVED_SETTINGS}), + ), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + serialized = json.dumps(response.model_dump()) + assert "super-secret-redis-pw" not in serialized + assert "super-secret-sentinel-pw" not in serialized + + assert response.values["password"] == _REDACTED_VALUE + assert response.values["sentinel_password"] == _REDACTED_VALUE + assert response.values["url"] == _REDACTED_VALUE + assert response.values["host"] == "coord-redis.example.com" + assert response.values["port"] == 6379 + + # field metadata is hydrated with the same redacted values + by_name = {field.field_name: field for field in response.fields} + assert by_name["password"].field_value == _REDACTED_VALUE + assert by_name["host"].field_value == "coord-redis.example.com" + + +@pytest.mark.asyncio +async def test_get_source_is_coordination_redis_when_block_present(monkeypatch): + """An explicit block wins even when a Redis cache backend and REDIS_* env both exist.""" + monkeypatch.setattr(litellm, "cache", MagicMock(cache=MagicMock(spec=RedisCache))) + monkeypatch.setenv("REDIS_HOST", "env-redis") + + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + _prisma_with_general_settings({"coordination_redis": _SAVED_SETTINGS}), + ), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source == "coordination_redis" + + +@pytest.mark.asyncio +async def test_get_source_reads_block_from_yaml_config_when_db_row_absent(monkeypatch): + """A block set in config.yaml (not the DB) still reports source=coordination_redis.""" + monkeypatch.setattr(litellm, "cache", None) + monkeypatch.delenv("REDIS_HOST", raising=False) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings(None)), + patch( + "litellm.proxy.proxy_server.proxy_config", + _proxy_config({"coordination_redis": {"host": "yaml-redis"}}), + ), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source == "coordination_redis" + assert response.values["host"] == "yaml-redis" + + +@pytest.mark.parametrize("cache_backend_cls", [RedisCache, RedisClusterCache]) +@pytest.mark.asyncio +async def test_get_source_is_cache_backend_when_no_block(monkeypatch, cache_backend_cls): + """With no explicit block, a plain-Redis response-cache backend is borrowed — + which beats the REDIS_* env fallback.""" + monkeypatch.setattr(litellm, "cache", MagicMock(cache=MagicMock(spec=cache_backend_cls))) + monkeypatch.setenv("REDIS_HOST", "env-redis") + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source == "cache_backend" + assert response.values == {} + + +@pytest.mark.asyncio +async def test_get_source_is_environment_when_no_block_and_non_redis_cache(monkeypatch): + """A non-Redis cache backend falls through to the REDIS_* env fallback.""" + monkeypatch.setattr(litellm, "cache", MagicMock(cache=MagicMock())) + monkeypatch.setenv("REDIS_HOST", "env-redis") + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source == "environment" + + +@pytest.mark.asyncio +async def test_get_source_is_none_when_nothing_configured(monkeypatch): + monkeypatch.setattr(litellm, "cache", None) + for env_var in ("REDIS_HOST", "REDIS_URL", "REDIS_CLUSTER_NODES", "REDIS_SENTINEL_NODES"): + monkeypatch.delenv(env_var, raising=False) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source is None + + +@pytest.mark.asyncio +async def test_get_source_does_not_build_a_client(monkeypatch): + """The env-fallback probe is read-only: no Redis client is constructed on GET.""" + monkeypatch.setattr(litellm, "cache", None) + monkeypatch.setenv("REDIS_HOST", "env-redis") + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server._build_redis_usage_cache") as mock_build, + ): + response = await get_coordination_redis_settings(user_api_key_dict=_admin_auth()) + + assert response.source == "environment" + mock_build.assert_not_called() + + +@pytest.mark.asyncio +async def test_get_rejects_non_admin(): + with pytest.raises(HTTPException) as exc_info: + await get_coordination_redis_settings( + user_api_key_dict=UserAPIKeyAuth(api_key="hashed", user_role=LitellmUserRoles.INTERNAL_USER) + ) + assert exc_info.value.status_code == 403 + + +def test_fields_cover_every_coordination_redis_param(): + """The declarative field list drives the Admin UI form; it must stay in sync + with the model the backend validates against.""" + from litellm.proxy._types import CoordinationRedisParams + + assert {field.field_name for field in COORDINATION_REDIS_SETTINGS_FIELDS} == set( + CoordinationRedisParams.model_fields.keys() + ) + + by_name = {field.field_name: field for field in COORDINATION_REDIS_SETTINGS_FIELDS} + assert by_name["startup_nodes"].section == "cluster" + assert by_name["sentinel_nodes"].section == "sentinel" + assert by_name["host"].section == "connection" + + +# ── POST /coordination_redis/settings ───────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_update_rejects_settings_without_a_connection_target(monkeypatch): + """A block with no host/url/startup_nodes/sentinel_nodes would blow up at + startup; reject it at write time and persist nothing.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + mock_prisma = _prisma_with_general_settings({}) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + ): + with pytest.raises(HTTPException) as exc_info: + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest(settings={"ssl": True, "service_name": "mymaster"}), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + assert exc_info.value.status_code == 400 + mock_prisma.db.litellm_config.upsert.assert_not_called() + + +@pytest.mark.asyncio +async def test_update_persists_into_the_general_settings_config_row(monkeypatch): + """Settings land under `general_settings.coordination_redis` in LiteLLM_Config + (the row startup merges over the yaml config), and sibling general_settings + keys survive the write.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + mock_prisma = _prisma_with_general_settings({"master_key": "sk-1234"}) + invalidated: list[str] = [] + + async def _capture_invalidate(param_name: str) -> None: + invalidated.append(param_name) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints.invalidate_config_param", + new=_capture_invalidate, + ), + ): + response = await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest( + settings={"host": "coord-redis.example.com", "port": 6379, "password": "pw"} + ), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + upsert_kwargs = mock_prisma.db.litellm_config.upsert.call_args.kwargs + assert upsert_kwargs["where"] == {"param_name": "general_settings"} + persisted = json.loads(upsert_kwargs["data"]["update"]["param_value"]) + assert persisted["coordination_redis"] == { + "host": "coord-redis.example.com", + "port": 6379, + "password": "pw", + } + assert persisted["master_key"] == "sk-1234" + assert invalidated == ["general_settings"] + + # the response echoes the saved settings back redacted + assert response["settings"]["password"] == _REDACTED_VALUE + assert response["settings"]["host"] == "coord-redis.example.com" + + +@pytest.mark.asyncio +async def test_update_persists_os_environ_refs_verbatim(monkeypatch): + """`os.environ/VAR` refs are resolved only to validate; the ref itself is what + gets stored, so the credential never lands in the DB.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + monkeypatch.setenv("MY_REDIS_HOST", "resolved-host") + mock_prisma = _prisma_with_general_settings({}) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints.invalidate_config_param", + new=AsyncMock(), + ), + ): + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest(settings={"host": "os.environ/MY_REDIS_HOST"}), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + persisted = json.loads(mock_prisma.db.litellm_config.upsert.call_args.kwargs["data"]["update"]["param_value"]) + assert persisted["coordination_redis"] == {"host": "os.environ/MY_REDIS_HOST"} + + +@pytest.mark.asyncio +async def test_update_keeps_saved_credential_when_client_echoes_the_redaction_marker(monkeypatch): + """The UI reads settings back redacted; re-submitting them must not persist + `***REDACTED***` as the password.""" + monkeypatch.setattr(litellm, "store_audit_logs", False) + mock_prisma = _prisma_with_general_settings({"coordination_redis": _SAVED_SETTINGS}) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints.invalidate_config_param", + new=AsyncMock(), + ), + ): + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest( + settings={"host": "new-host", "port": 6380, "password": _REDACTED_VALUE} + ), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + persisted = json.loads(mock_prisma.db.litellm_config.upsert.call_args.kwargs["data"]["update"]["param_value"]) + assert persisted["coordination_redis"]["password"] == "super-secret-redis-pw" + assert persisted["coordination_redis"]["host"] == "new-host" + + +@pytest.mark.asyncio +async def test_update_emits_audit_log_with_values_redacted(monkeypatch): + monkeypatch.setattr(litellm, "store_audit_logs", True) + mock_prisma = _prisma_with_general_settings({}) + audit_calls = [] + + async def capture(request_data): + audit_calls.append(request_data) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints.invalidate_config_param", + new=AsyncMock(), + ), + patch("litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", new=capture), + ): + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest( + settings={"host": "coord-redis.example.com", "password": "super-secret-redis-pw"} + ), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + for _ in range(3): + await asyncio.sleep(0) + + assert len(audit_calls) == 1 + log = audit_calls[0] + assert log.table_name == LitellmTableNames.CONFIG_TABLE_NAME + assert log.object_id == "coordination_redis" + assert log.action == "created" # no prior block → create + + after = json.loads(log.updated_values) + assert set(after["settings"].keys()) == {"host", "password"} + assert "super-secret-redis-pw" not in log.updated_values + assert "coord-redis.example.com" not in log.updated_values + + +@pytest.mark.asyncio +async def test_update_audit_action_is_updated_when_a_block_already_exists(monkeypatch): + monkeypatch.setattr(litellm, "store_audit_logs", True) + mock_prisma = _prisma_with_general_settings({"coordination_redis": {"host": "old-host"}}) + audit_calls = [] + + async def capture(request_data): + audit_calls.append(request_data) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server.store_model_in_db", True), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints.invalidate_config_param", + new=AsyncMock(), + ), + patch("litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", new=capture), + ): + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest(settings={"host": "new-host"}), + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + for _ in range(3): + await asyncio.sleep(0) + + assert audit_calls[0].action == "updated" + assert json.loads(audit_calls[0].before_value)["settings"] == {"host": _REDACTED_VALUE} + + +@pytest.mark.asyncio +async def test_update_rejects_non_admin(): + with pytest.raises(HTTPException) as exc_info: + await update_coordination_redis_settings( + request=CoordinationRedisSettingsRequest(settings={"host": "coord-redis.example.com"}), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed", user_role=LitellmUserRoles.INTERNAL_USER), + litellm_changed_by=None, + ) + assert exc_info.value.status_code == 403 + + +# ── POST /coordination_redis/settings/test ──────────────────────────────────── + + +@pytest.mark.asyncio +async def test_connection_test_returns_healthy_on_successful_ping(): + mock_client = MagicMock() + mock_client.ping = AsyncMock(return_value=True) + mock_client.disconnect = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server._build_redis_usage_cache", return_value=mock_client) as mock_build, + ): + response = await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest( + settings={"host": "coord-redis.example.com", "port": 6379, "password": "pw"} + ), + user_api_key_dict=_admin_auth(), + ) + + assert response.status == "healthy" + assert response.error is None + assert mock_build.call_args.args[0] == {"host": "coord-redis.example.com", "port": 6379, "password": "pw"} + mock_client.ping.assert_awaited_once() + mock_client.disconnect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_connection_test_reports_unhealthy_without_leaking_the_password(): + """Redis client errors echo the connection url back; the password must be + scrubbed out of the error the admin sees.""" + mock_client = MagicMock() + mock_client.ping = AsyncMock( + side_effect=ConnectionError("Error connecting to redis://:super-secret-redis-pw@coord-redis.example.com:6379") + ) + mock_client.disconnect = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server._build_redis_usage_cache", return_value=mock_client), + ): + response = await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest( + settings={ + "host": "coord-redis.example.com", + "url": "redis://:super-secret-redis-pw@coord-redis.example.com:6379", + "password": "super-secret-redis-pw", + } + ), + user_api_key_dict=_admin_auth(), + ) + + assert response.status == "unhealthy" + assert response.error is not None + assert "super-secret-redis-pw" not in response.error + assert _REDACTED_VALUE in response.error + mock_client.disconnect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_connection_test_uses_the_saved_password_for_a_redacted_field(): + """An admin re-testing settings read back from GET sends `***REDACTED***`; + the saved credential is what actually gets dialed.""" + mock_client = MagicMock() + mock_client.ping = AsyncMock(return_value=True) + mock_client.disconnect = AsyncMock() + + with ( + patch( + "litellm.proxy.proxy_server.prisma_client", + _prisma_with_general_settings({"coordination_redis": _SAVED_SETTINGS}), + ), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server._build_redis_usage_cache", return_value=mock_client) as mock_build, + ): + response = await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest( + settings={"host": "coord-redis.example.com", "password": _REDACTED_VALUE} + ), + user_api_key_dict=_admin_auth(), + ) + + assert response.status == "healthy" + assert mock_build.call_args.args[0]["password"] == "super-secret-redis-pw" + + +@pytest.mark.asyncio +async def test_connection_test_times_out_instead_of_hanging(): + async def _never_returns(): + await asyncio.sleep(60) + + mock_client = MagicMock() + mock_client.ping = MagicMock(side_effect=lambda: _never_returns()) + mock_client.disconnect = AsyncMock() + + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + patch("litellm.proxy.proxy_server._build_redis_usage_cache", return_value=mock_client), + patch( + "litellm.proxy.management_endpoints.coordination_redis_endpoints._PING_TIMEOUT_SECONDS", + 0.01, + ), + ): + response = await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest(settings={"host": "unreachable"}), + user_api_key_dict=_admin_auth(), + ) + + assert response.status == "unhealthy" + assert "timed out" in (response.error or "") + + +@pytest.mark.asyncio +async def test_connection_test_rejects_settings_without_a_connection_target(): + with ( + patch("litellm.proxy.proxy_server.prisma_client", _prisma_with_general_settings({})), + patch("litellm.proxy.proxy_server.proxy_config", _proxy_config()), + ): + with pytest.raises(HTTPException) as exc_info: + await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest(settings={"ssl": True}), + user_api_key_dict=_admin_auth(), + ) + + assert exc_info.value.status_code == 400 + + +@pytest.mark.asyncio +async def test_connection_test_rejects_non_admin(): + with pytest.raises(HTTPException) as exc_info: + await check_coordination_redis_connection( + request=CoordinationRedisSettingsRequest(settings={"host": "coord-redis.example.com"}), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed", user_role=LitellmUserRoles.INTERNAL_USER), + ) + assert exc_info.value.status_code == 403 diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index a2d186bcdbf..603d5cc15b7 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -26,6 +26,7 @@ sys.path.insert( import litellm import litellm.proxy.proxy_server as proxy_server_module from litellm.caching.caching import RedisCache +from litellm.caching.redis_cluster_cache import RedisClusterCache from litellm.caching.dual_cache import DualCache from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -9396,11 +9397,9 @@ def _run_init_cache_with_backend(cache_backend, redis_env_kwargs): patch("litellm.Cache", return_value=mock_litellm_cache), ): litellm.cache = None - proxy_server_module.ProxyConfig()._init_cache( - cache_params={"type": "qdrant-semantic"} - ) + resolved = proxy_server_module.ProxyConfig()._init_cache(cache_params={"type": "qdrant-semantic"}) return ( - proxy_server_module.redis_usage_cache, + resolved, fresh_spend_cache.redis_cache, fresh_config_cache.redis_cache, ) @@ -9448,3 +9447,242 @@ def test_init_cache_redis_backend_reuses_cache_backend_over_environment(): assert usage_cache is redis_backend assert usage_cache.init_kwargs["host"] == "cache-params-host" assert spend_redis is redis_backend + + +class _EnvBuiltClusterCache(RedisClusterCache): + """RedisClusterCache stand-in that records constructor kwargs and never + opens a network connection.""" + + def __init__(self, **kwargs): + self.init_kwargs = kwargs + + +def _run_init_coordination_redis(config, env=None): + """Run ProxyConfig._init_coordination_redis against a stubbed module state, + returning (redis_usage_cache, spend_counter redis, config-cache redis).""" + fresh_spend_cache = DualCache() + fresh_config_cache = types.SimpleNamespace(redis_cache=None) + + with ( + patch.object(proxy_server_module, "redis_usage_cache", None), + patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), + patch.object(proxy_server_module, "user_api_key_cache", DualCache()), + patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + mock.patch.dict(os.environ, env or {}, clear=False), + ): + built = proxy_server_module.ProxyConfig()._init_coordination_redis(config=config) + return ( + built, + fresh_spend_cache.redis_cache, + fresh_config_cache.redis_cache, + ) + + +def test_init_coordination_redis_explicit_block_builds_standalone_client(): + """general_settings.coordination_redis must build the coordination Redis + even when no response cache is configured at all, and attach it to the + spend counter and config caches.""" + usage_cache, spend_redis, config_redis = _run_init_coordination_redis( + config={"general_settings": {"coordination_redis": {"host": "coord-host", "port": 6380}}}, + ) + + assert isinstance(usage_cache, _EnvBuiltRedisCache) + assert usage_cache.init_kwargs["host"] == "coord-host" + assert usage_cache.init_kwargs["port"] == 6380 + assert spend_redis is usage_cache + assert config_redis is usage_cache + + +def test_init_coordination_redis_resolves_os_environ_references(): + """os.environ/ values inside the coordination_redis block must be resolved + the same way cache_params values are.""" + usage_cache, _, _ = _run_init_coordination_redis( + config={"general_settings": {"coordination_redis": {"host": "os.environ/COORD_REDIS_HOST"}}}, + env={"COORD_REDIS_HOST": "resolved-host"}, + ) + + assert usage_cache.init_kwargs["host"] == "resolved-host" + + +def test_init_coordination_redis_startup_nodes_builds_cluster_client(): + """A coordination_redis block with startup_nodes must construct a cluster + client, so cluster-aware consumers (v3 rate limiter) take the cluster path.""" + usage_cache, _, _ = _run_init_coordination_redis( + config={ + "general_settings": { + "coordination_redis": {"startup_nodes": [{"host": "node-1", "port": 7000}]} + } + }, + ) + + assert isinstance(usage_cache, _EnvBuiltClusterCache) + assert usage_cache.init_kwargs["startup_nodes"] == [{"host": "node-1", "port": 7000}] + + +def test_init_coordination_redis_without_connection_target_raises(): + """A coordination_redis block with no host, url, startup_nodes, or + sentinel_nodes is a config error and must fail startup loudly instead of + silently running without coordination.""" + with pytest.raises(ValueError, match="connection target"): + _run_init_coordination_redis( + config={"general_settings": {"coordination_redis": {"ssl": True}}}, + ) + + +def test_init_coordination_redis_non_mapping_block_raises(): + """A scalar coordination_redis value is a config error.""" + with pytest.raises(ValueError, match="mapping"): + _run_init_coordination_redis( + config={"general_settings": {"coordination_redis": "redis://host:6379"}}, + ) + + +def test_init_coordination_redis_absent_leaves_usage_cache_unset(): + """Without the block, nothing changes: the coordination Redis stays unset + for the downstream borrow / env fallback logic to decide.""" + usage_cache, spend_redis, _ = _run_init_coordination_redis( + config={"general_settings": {}}, + ) + + assert usage_cache is None + assert spend_redis is None + + +def test_explicit_coordination_redis_takes_precedence_over_cache_backend(): + """When both an explicit coordination_redis block and a plain-Redis + response cache are configured, the explicit block must win; the cache + backend must not overwrite it.""" + fresh_spend_cache = DualCache() + fresh_config_cache = types.SimpleNamespace(redis_cache=None) + cache_backend = _EnvBuiltRedisCache(host="cache-backend-host") + mock_litellm_cache = MagicMock() + mock_litellm_cache.cache = cache_backend + + with ( + patch.object(proxy_server_module, "redis_usage_cache", None), + patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), + patch.object(proxy_server_module, "user_api_key_cache", DualCache()), + patch.object(proxy_server_module, "llm_router", None), + patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + patch("litellm.Cache", return_value=mock_litellm_cache), + ): + litellm.cache = None + proxy_config = proxy_server_module.ProxyConfig() + built = proxy_config._init_coordination_redis( + config={"general_settings": {"coordination_redis": {"host": "explicit-coord-host"}}} + ) + assert built is not None + proxy_server_module.redis_usage_cache = built + usage_cache = proxy_config._init_cache(cache_params={"type": "redis"}) + + assert isinstance(usage_cache, _EnvBuiltRedisCache) + assert usage_cache is not cache_backend + assert usage_cache.init_kwargs["host"] == "explicit-coord-host" + assert fresh_spend_cache.redis_cache is usage_cache + + +def test_env_fallback_builds_cluster_client_from_cluster_nodes_env(): + """A deployment whose only Redis env is REDIS_CLUSTER_NODES must still get + a coordination Redis from the env fallback, and it must be a cluster + client so cluster-aware consumers take the cluster path.""" + nodes = '[{"host": "cnode-1", "port": 7000}]' + with ( + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + patch("litellm._redis._redis_kwargs_from_environment", return_value={}), + mock.patch.dict(os.environ, {"REDIS_CLUSTER_NODES": nodes}, clear=False), + ): + result = proxy_server_module._build_redis_usage_cache_from_environment() + + assert isinstance(result, _EnvBuiltClusterCache) + assert result.init_kwargs["startup_nodes"] == [{"host": "cnode-1", "port": 7000}] + + +def test_env_fallback_builds_client_from_sentinel_nodes_env(): + """A sentinel-only environment (REDIS_SENTINEL_NODES, no host or url) must + also produce a coordination Redis from the env fallback.""" + with ( + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + patch("litellm._redis._redis_kwargs_from_environment", return_value={}), + mock.patch.dict(os.environ, {"REDIS_SENTINEL_NODES": '[["s1", 26379]]'}, clear=False), + ): + result = proxy_server_module._build_redis_usage_cache_from_environment() + + assert isinstance(result, _EnvBuiltRedisCache) + + +@pytest.mark.asyncio +async def test_startup_applies_coordination_redis_saved_in_database(): + """A coordination_redis block saved from the admin UI lives only in the + database, so startup must read it and build the coordination Redis from it. + Without this the save endpoint's "restart to apply" promise is false and the + proxy silently coordinates in per-pod memory.""" + fresh_spend_cache = DualCache() + fresh_config_cache = types.SimpleNamespace(redis_cache=None) + + with ( + patch.object(proxy_server_module, "spend_counter_cache", fresh_spend_cache), + patch.object(proxy_server_module, "user_api_key_cache", DualCache()), + patch.object(proxy_server_module, "litellm_config_cache", fresh_config_cache), + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object(proxy_server_module, "RedisClusterCache", _EnvBuiltClusterCache), + patch.object( + proxy_server_module, + "get_persisted_coordination_redis_settings", + AsyncMock(return_value={"host": "db-host", "port": 6381}), + ), + ): + result = await proxy_server_module.ProxyStartupEvent._init_coordination_redis_from_db( + litellm_settings={}, + llm_router=None, + ) + + assert isinstance(result, _EnvBuiltRedisCache) + assert result.init_kwargs["host"] == "db-host" + assert fresh_spend_cache.redis_cache is result + assert fresh_config_cache.redis_cache is result + + +@pytest.mark.asyncio +async def test_startup_ignores_database_coordination_redis_without_connection_target(): + """A persisted block with no host/url/cluster/sentinel must be ignored rather + than crashing startup or building a client that cannot connect.""" + with ( + patch.object(proxy_server_module, "spend_counter_cache", DualCache()), + patch.object(proxy_server_module, "litellm_config_cache", types.SimpleNamespace(redis_cache=None)), + patch.object(proxy_server_module, "RedisCache", _EnvBuiltRedisCache), + patch.object( + proxy_server_module, + "get_persisted_coordination_redis_settings", + AsyncMock(return_value={"ssl": True}), + ), + ): + result = await proxy_server_module.ProxyStartupEvent._init_coordination_redis_from_db( + litellm_settings={}, + llm_router=None, + ) + + assert result is None + + +@pytest.mark.asyncio +async def test_startup_survives_database_read_failure_for_coordination_redis(): + """A config-row read failure must not block proxy startup.""" + with ( + patch.object( + proxy_server_module, + "get_persisted_coordination_redis_settings", + AsyncMock(side_effect=RuntimeError("db unreachable")), + ), + ): + result = await proxy_server_module.ProxyStartupEvent._init_coordination_redis_from_db( + litellm_settings={}, + llm_router=None, + ) + + assert result is None diff --git a/tests/test_litellm/test_redis.py b/tests/test_litellm/test_redis.py index 0081e1c819f..305b402821b 100644 --- a/tests/test_litellm/test_redis.py +++ b/tests/test_litellm/test_redis.py @@ -629,3 +629,46 @@ def test_sync_client_url_used_when_no_cluster(mock_from_url, monkeypatch): get_redis_client() mock_from_url.assert_called_once() + + +@patch("litellm._redis.redis.Redis.from_url") +def test_explicit_host_outranks_environment_redis_url(mock_from_url, monkeypatch): + """ + An explicitly configured host must win over REDIS_URL in the environment. + + Otherwise the url branch strips the caller's host/port and the client + silently connects to whatever REDIS_URL names, so an explicit config block + (or a connection test typed into the admin UI) targets the wrong server. + """ + monkeypatch.setenv("REDIS_URL", "redis://env-host:6379") + monkeypatch.delenv("REDIS_CLUSTER_NODES", raising=False) + + client = get_redis_client(host="explicit-host", port=6380) + + mock_from_url.assert_not_called() + assert client.connection_pool.connection_kwargs["host"] == "explicit-host" + assert client.connection_pool.connection_kwargs["port"] == 6380 + + +@patch("litellm._redis.redis.Redis.from_url") +def test_explicit_url_still_wins_over_environment_host(mock_from_url, monkeypatch): + """An explicit url argument keeps taking the from_url path.""" + monkeypatch.setenv("REDIS_HOST", "env-host") + monkeypatch.setenv("REDIS_PORT", "6379") + monkeypatch.delenv("REDIS_CLUSTER_NODES", raising=False) + + get_redis_client(url="redis://explicit-host:6380") + + mock_from_url.assert_called_once() + assert mock_from_url.call_args.kwargs["url"] == "redis://explicit-host:6380" + + +@patch("litellm._redis.redis.Redis.from_url") +def test_environment_redis_url_used_when_caller_names_no_target(mock_from_url, monkeypatch): + """With no caller-supplied connection target, REDIS_URL still drives the client.""" + monkeypatch.setenv("REDIS_URL", "redis://env-host:6379") + monkeypatch.delenv("REDIS_CLUSTER_NODES", raising=False) + + get_redis_client() + + mock_from_url.assert_called_once() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx index 944983da0e1..c53e6318d6b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx @@ -25,6 +25,7 @@ import { adminGlobalCacheActivity, cachingHealthCheckCall } from "@/components/n // Import the new component import { CacheHealthTab } from "./cache_health"; import CacheSettings from "./cache_settings"; +import CoordinationRedisSettings from "./coordination_redis_settings"; const formatDateWithoutTZ = (date: Date | undefined) => { if (!date) return undefined; @@ -264,6 +265,7 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole Cache Analytics Cache Health Cache Settings + Coordination Redis
@@ -383,6 +385,9 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole + + + ); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFieldSection.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFieldSection.tsx new file mode 100644 index 00000000000..af807926ed5 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFieldSection.tsx @@ -0,0 +1,46 @@ +import React from "react"; +import CoordinationRedisFormField from "./CoordinationRedisFormField"; +import { fieldsForSection } from "./coordinationRedisUtils"; +import { CoordinationRedisType, CoordinationSection } from "./coordinationRedisFields"; + +interface CoordinationRedisFieldSectionProps { + title: string; + section: CoordinationSection; + redisType: CoordinationRedisType; + configuredSecrets: ReadonlySet; + gridCols?: string; + headingLevel?: "h4" | "h5"; +} + +const CoordinationRedisFieldSection: React.FC = ({ + title, + section, + redisType, + configuredSecrets, + gridCols = "grid-cols-1 gap-6 sm:grid-cols-2", + headingLevel = "h4", +}) => { + const fields = fieldsForSection(section, redisType); + if (fields.length === 0) { + return null; + } + + const Heading = headingLevel; + + return ( +
+ {title} +
+ {fields.map((field) => ( + + ))} +
+
+ ); +}; + +export default CoordinationRedisFieldSection; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFormField.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFormField.tsx new file mode 100644 index 00000000000..50c2a39567a --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisFormField.tsx @@ -0,0 +1,39 @@ +import { Form, Input, Switch } from "antd"; +import React from "react"; +import { CoordinationField } from "./coordinationRedisFields"; + +export const SECRET_ALREADY_SET_PLACEHOLDER = "Already set. Enter a new value to replace it."; + +interface CoordinationRedisFormFieldProps { + field: CoordinationField; + isSecretConfigured: boolean; +} + +const renderControl = (field: CoordinationField, placeholder: string): React.ReactNode => { + switch (field.type) { + case "boolean": + return ; + case "password": + return ; + case "integer": + return ; + case "list": + return ; + default: + return ; + } +}; + +const CoordinationRedisFormField: React.FC = ({ field, isSecretConfigured }) => ( + + {renderControl(field, isSecretConfigured ? SECRET_ALREADY_SET_PLACEHOLDER : field.helpText)} + +); + +export default CoordinationRedisFormField; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisTypeSelector.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisTypeSelector.tsx new file mode 100644 index 00000000000..daab8505890 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/coordination_redis_settings/CoordinationRedisTypeSelector.tsx @@ -0,0 +1,33 @@ +import React from "react"; +import { Select } from "antd"; +import { + COORDINATION_REDIS_TYPES, + COORDINATION_REDIS_TYPE_DESCRIPTIONS, + COORDINATION_REDIS_TYPE_LABELS, + CoordinationRedisType, +} from "./coordinationRedisFields"; + +interface CoordinationRedisTypeSelectorProps { + redisType: CoordinationRedisType; + onTypeChange: (type: CoordinationRedisType) => void; +} + +const OPTIONS = COORDINATION_REDIS_TYPES.map((type) => ({ value: type, label: COORDINATION_REDIS_TYPE_LABELS[type] })); + +const CoordinationRedisTypeSelector: React.FC = ({ redisType, onTypeChange }) => ( +
+ +