mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-15 04:19:08 +00:00
feat(proxy): add deployment warning endpoints with instance-wide ignore and snooze (#46347)
* feat(proxy): add deployment warning endpoints with instance-wide ignore and snooze
GET /health/deployment_warnings returns the active deployment warnings
(env credential login, license expiry, no Redis on multiple workers,
detailed debug logging) with a severity and ignore state, for proxy admins
and proxy admin viewers. PATCH /health/deployment_warnings/{warning_id}
lets a proxy admin ignore a warning forever or snooze it for up to 30 days,
or restore it. Ignores live in one LiteLLM_UISettings row, apply to every
admin, and are written to the config audit log. Expired snoozes drop out
on read, and LITELLM_DISABLE_NO_REDIS_WARNING is reported as an env var
ignore. /health/readiness/details is unchanged
* fix(proxy): keep readiness Redis short-circuit ahead of the env lookup and harden ignore storage
Readiness details again answers from Redis before reading
LITELLM_DISABLE_NO_REDIS_WARNING, so proxies with Redis and a secret
manager do not fetch it on every poll. Stored ignores require timezone
aware datetimes and are read from the writer before a rewrite
* fix(proxy): keep deployment warning ignores readable in the config audit log
The audit payload keyed ignores by warning id, so the secret matcher
redacted the env_credential_login entry and the audit row lost who
snoozed it and until when. Log a list of id and ignore entries instead
This commit is contained in:
yuneng-jiang
GitHub
parent
d0f3a2dfdb
commit
a0ac9b4f1f
8 changed files with 1081 additions and 13 deletions
|
|
@ -1464,7 +1464,7 @@ async def shared_health_check_status_endpoint(
|
|||
)
|
||||
|
||||
|
||||
def _read_license_data() -> EnterpriseLicenseData | None:
|
||||
def read_license_data() -> EnterpriseLicenseData | None:
|
||||
from litellm.proxy.proxy_server import _license_check, premium_user_data
|
||||
|
||||
license_data: EnterpriseLicenseData | None = premium_user_data or _license_check.airgapped_license_data
|
||||
|
|
@ -1509,7 +1509,7 @@ async def health_license_endpoint(
|
|||
"""Return metadata about the configured LiteLLM license without exposing the key."""
|
||||
from litellm.proxy.proxy_server import _license_check, premium_user
|
||||
|
||||
license_data: Final = _read_license_data()
|
||||
license_data: Final = read_license_data()
|
||||
has_license: Final = bool(getattr(_license_check, "license_str", None))
|
||||
license_type: Final = "enterprise" if premium_user else "community"
|
||||
|
||||
|
|
@ -1703,9 +1703,9 @@ def callback_name(callback):
|
|||
DISABLE_NO_REDIS_WARNING_ENV_VAR: Final = "LITELLM_DISABLE_NO_REDIS_WARNING"
|
||||
|
||||
|
||||
async def _show_no_redis_warning() -> bool:
|
||||
async def no_redis_on_multiple_workers() -> bool:
|
||||
"""
|
||||
Whether the UI should warn that no Redis is configured.
|
||||
Whether this proxy runs without Redis on more than one worker.
|
||||
|
||||
Redis is what makes rate limits, budgets, router state, and cache
|
||||
invalidation consistent across workers, so a proxy running without it is
|
||||
|
|
@ -1716,22 +1716,34 @@ async def _show_no_redis_warning() -> bool:
|
|||
routing on its own. A deployment whose worker-heartbeat census proves it
|
||||
is exactly one worker needs no cross-worker coordination, so it never
|
||||
warns; when the census is unavailable or shows more than one worker, the
|
||||
warning stands unless LITELLM_DISABLE_NO_REDIS_WARNING=true silences it.
|
||||
warning stands.
|
||||
"""
|
||||
from litellm.proxy.proxy_server import llm_router, prisma_client, redis_usage_cache
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if redis_usage_cache is not None:
|
||||
return False
|
||||
if llm_router is not None and llm_router.cache.redis_cache is not None:
|
||||
return False
|
||||
if get_secret_bool(DISABLE_NO_REDIS_WARNING_ENV_VAR, False) is True:
|
||||
if _redis_configured():
|
||||
return False
|
||||
if prisma_client is None:
|
||||
return True
|
||||
return await count_live_proxy_workers(prisma_client) != 1
|
||||
|
||||
|
||||
def _show_env_credential_login_warning() -> bool:
|
||||
def _redis_configured() -> bool:
|
||||
from litellm.proxy.proxy_server import llm_router, redis_usage_cache
|
||||
|
||||
return redis_usage_cache is not None or (llm_router is not None and llm_router.cache.redis_cache is not None)
|
||||
|
||||
|
||||
def no_redis_warning_disabled_by_env() -> bool:
|
||||
return get_secret_bool(DISABLE_NO_REDIS_WARNING_ENV_VAR, False) is True
|
||||
|
||||
|
||||
async def _show_no_redis_warning() -> bool:
|
||||
if _redis_configured() or no_redis_warning_disabled_by_env():
|
||||
return False
|
||||
return await no_redis_on_multiple_workers()
|
||||
|
||||
|
||||
def env_credential_login_enabled() -> bool:
|
||||
from litellm.proxy.auth.login_utils import is_env_credential_login_enabled
|
||||
from litellm.proxy.proxy_server import general_settings
|
||||
|
||||
|
|
@ -1779,7 +1791,7 @@ async def _get_health_readiness_details(
|
|||
log_level_name: Final = logging.getLevelName(verbose_logger.getEffectiveLevel())
|
||||
is_detailed_debug: Final = verbose_logger.isEnabledFor(logging.DEBUG)
|
||||
show_no_redis_warning: Final = await _show_no_redis_warning()
|
||||
show_env_credential_login_warning: Final = _show_env_credential_login_warning()
|
||||
show_env_credential_login_warning: Final = env_credential_login_enabled()
|
||||
|
||||
# check DB
|
||||
if prisma_client is not None: # if db passed in, check if it's connected
|
||||
|
|
|
|||
350
litellm/proxy/health_endpoints/deployment_warning_endpoints.py
Normal file
350
litellm/proxy/health_endpoints/deployment_warning_endpoints.py
Normal file
|
|
@ -0,0 +1,350 @@
|
|||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, Literal, Protocol, TypeAlias
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import AwareDatetime, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError, model_validator
|
||||
|
||||
from litellm._logging import verbose_logger, verbose_proxy_logger
|
||||
from litellm.constants import LITELLM_PROXY_ADMIN_NAME
|
||||
from litellm.proxy._types import AUDIT_ACTIONS, LitellmTableNames, LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.health_endpoints._health_endpoints import (
|
||||
DISABLE_NO_REDIS_WARNING_ENV_VAR,
|
||||
env_credential_login_enabled,
|
||||
no_redis_on_multiple_workers,
|
||||
no_redis_warning_disabled_by_env,
|
||||
read_license_data,
|
||||
)
|
||||
from litellm.repositories.deployment_warning_ignore_repository import (
|
||||
DEPLOYMENT_WARNING_IGNORES_ROW_ID,
|
||||
DeploymentWarningIgnoreRepository,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
ENV_CREDENTIAL_LOGIN_WARNING_ID: Final = "env_credential_login"
|
||||
NO_REDIS_WARNING_ID: Final = "no_redis"
|
||||
DETAILED_DEBUG_WARNING_ID: Final = "detailed_debug"
|
||||
LICENSE_EXPIRY_WARNING_ID_PREFIX: Final = "license_expiry:"
|
||||
LICENSE_EXPIRY_WARNING_DAYS: Final = 30
|
||||
LICENSE_EXPIRY_CRITICAL_DAYS: Final = 7
|
||||
MAX_SNOOZE_SECONDS: Final = 30 * 24 * 60 * 60
|
||||
|
||||
_STATIC_WARNING_IDS: Final = frozenset(
|
||||
{ENV_CREDENTIAL_LOGIN_WARNING_ID, NO_REDIS_WARNING_ID, DETAILED_DEBUG_WARNING_ID}
|
||||
)
|
||||
|
||||
WarningSeverity: TypeAlias = Literal["critical", "warning"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WarningCondition:
|
||||
id: str
|
||||
severity: WarningSeverity
|
||||
env_var_ignore: str | None = None
|
||||
|
||||
|
||||
class AdminIgnore(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
source: Literal["admin"] = "admin"
|
||||
ignored_by: str
|
||||
ignored_at: AwareDatetime
|
||||
ignored_until: AwareDatetime | None = Field(
|
||||
default=None,
|
||||
description="When the snooze ends. Null means the warning is ignored until an admin restores it.",
|
||||
)
|
||||
|
||||
|
||||
class EnvVarIgnore(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
source: Literal["env_var"] = "env_var"
|
||||
env_var: str
|
||||
|
||||
|
||||
class DeploymentWarning(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str
|
||||
severity: WarningSeverity
|
||||
ignore: Annotated[AdminIgnore | EnvVarIgnore, Field(discriminator="source")] | None = None
|
||||
|
||||
|
||||
class DeploymentWarningsResponse(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
warnings: tuple[DeploymentWarning, ...]
|
||||
ignore_available: bool = Field(description="False when no database is connected, so warnings cannot be ignored.")
|
||||
|
||||
|
||||
class DeploymentWarningIgnoreUpdate(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
ignored: bool = Field(description="True ignores the warning for every admin, false restores it.")
|
||||
snooze_seconds: int | None = Field(
|
||||
default=None,
|
||||
gt=0,
|
||||
le=MAX_SNOOZE_SECONDS,
|
||||
description="How long to ignore the warning, up to 30 days. Leave it out to ignore it forever.",
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _snooze_only_when_ignoring(self) -> "DeploymentWarningIgnoreUpdate":
|
||||
if self.snooze_seconds is not None and not self.ignored:
|
||||
raise ValueError("snooze_seconds is only valid when ignored is true")
|
||||
return self
|
||||
|
||||
|
||||
class DeploymentWarningIgnoreResponse(LiteLLMBaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str
|
||||
ignore: AdminIgnore | None
|
||||
|
||||
|
||||
class DeploymentWarningIgnoreStore(Protocol):
|
||||
async def get_raw_ignores(self) -> object: ...
|
||||
|
||||
async def upsert_ignores(self, payload: str) -> None: ...
|
||||
|
||||
|
||||
class ConfigAuditLogger(Protocol):
|
||||
async def __call__(
|
||||
self,
|
||||
param_name: str,
|
||||
action: AUDIT_ACTIONS,
|
||||
before_value: JsonValue | None,
|
||||
after_value: JsonValue | None,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
table_name: LitellmTableNames = ...,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
_IGNORES_ADAPTER: Final[TypeAdapter[Mapping[str, AdminIgnore]]] = TypeAdapter(Mapping[str, AdminIgnore])
|
||||
_JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
|
||||
_AUDIT_ENTRIES_ADAPTER: Final[TypeAdapter[tuple[DeploymentWarningIgnoreResponse, ...]]] = TypeAdapter(
|
||||
tuple[DeploymentWarningIgnoreResponse, ...]
|
||||
)
|
||||
|
||||
|
||||
def license_expiry_condition(expiration_date: str | None, now: datetime) -> WarningCondition | None:
|
||||
if not expiration_date:
|
||||
return None
|
||||
try:
|
||||
expires_on: Final = date.fromisoformat(expiration_date)
|
||||
except ValueError:
|
||||
return None
|
||||
days_left: Final = (expires_on - now.date()).days
|
||||
if days_left > LICENSE_EXPIRY_WARNING_DAYS:
|
||||
return None
|
||||
return WarningCondition(
|
||||
id=f"{LICENSE_EXPIRY_WARNING_ID_PREFIX}{expires_on.isoformat()}",
|
||||
severity="critical" if days_left <= LICENSE_EXPIRY_CRITICAL_DAYS else "warning",
|
||||
)
|
||||
|
||||
|
||||
def deployment_warning_conditions(
|
||||
*,
|
||||
env_credential_login_enabled: bool,
|
||||
license_expiration_date: str | None,
|
||||
no_redis_on_multiple_workers: bool,
|
||||
no_redis_warning_disabled_by_env: bool,
|
||||
detailed_debug: bool,
|
||||
now: datetime,
|
||||
) -> tuple[WarningCondition, ...]:
|
||||
no_redis_env_var: Final = DISABLE_NO_REDIS_WARNING_ENV_VAR if no_redis_warning_disabled_by_env else None
|
||||
candidates: Final = (
|
||||
WarningCondition(id=ENV_CREDENTIAL_LOGIN_WARNING_ID, severity="critical")
|
||||
if env_credential_login_enabled
|
||||
else None,
|
||||
license_expiry_condition(license_expiration_date, now),
|
||||
WarningCondition(id=NO_REDIS_WARNING_ID, severity="warning", env_var_ignore=no_redis_env_var)
|
||||
if no_redis_on_multiple_workers
|
||||
else None,
|
||||
WarningCondition(id=DETAILED_DEBUG_WARNING_ID, severity="warning") if detailed_debug else None,
|
||||
)
|
||||
return tuple(condition for condition in candidates if condition is not None)
|
||||
|
||||
|
||||
def is_known_warning_id(warning_id: str) -> bool:
|
||||
if warning_id in _STATIC_WARNING_IDS:
|
||||
return True
|
||||
if not warning_id.startswith(LICENSE_EXPIRY_WARNING_ID_PREFIX):
|
||||
return False
|
||||
expiration_date: Final = warning_id.removeprefix(LICENSE_EXPIRY_WARNING_ID_PREFIX)
|
||||
try:
|
||||
return date.fromisoformat(expiration_date).isoformat() == expiration_date
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _parse_stored_ignores(raw: object) -> Mapping[str, AdminIgnore]:
|
||||
if raw is None:
|
||||
return MappingProxyType({})
|
||||
try:
|
||||
return MappingProxyType(_IGNORES_ADAPTER.validate_python(json.loads(raw) if isinstance(raw, str) else raw))
|
||||
except (json.JSONDecodeError, ValidationError):
|
||||
verbose_proxy_logger.warning("Ignoring unreadable %s row", DEPLOYMENT_WARNING_IGNORES_ROW_ID)
|
||||
return MappingProxyType({})
|
||||
|
||||
|
||||
async def _read_unexpired_ignores(store: DeploymentWarningIgnoreStore, now: datetime) -> Mapping[str, AdminIgnore]:
|
||||
stored: Final = _parse_stored_ignores(await store.get_raw_ignores())
|
||||
return MappingProxyType(
|
||||
{
|
||||
warning_id: ignore
|
||||
for warning_id, ignore in stored.items()
|
||||
if ignore.ignored_until is None or ignore.ignored_until > now
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _resolve_ignore(
|
||||
condition: WarningCondition, ignores: Mapping[str, AdminIgnore]
|
||||
) -> AdminIgnore | EnvVarIgnore | None:
|
||||
if condition.env_var_ignore is not None:
|
||||
return EnvVarIgnore(env_var=condition.env_var_ignore)
|
||||
return ignores.get(condition.id)
|
||||
|
||||
|
||||
def _ignores_as_audit_entries(ignores: Mapping[str, AdminIgnore]) -> JsonValue:
|
||||
entries: Final = tuple(
|
||||
DeploymentWarningIgnoreResponse(id=warning_id, ignore=ignores[warning_id]) for warning_id in sorted(ignores)
|
||||
)
|
||||
return _JSON_VALUE_ADAPTER.validate_json(_AUDIT_ENTRIES_ADAPTER.dump_json(entries))
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def deployment_warning_ignore_store() -> DeploymentWarningIgnoreStore | None:
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if prisma_client is None:
|
||||
return None
|
||||
return DeploymentWarningIgnoreRepository(prisma_client, use_writer=True)
|
||||
|
||||
|
||||
def config_audit_logger() -> ConfigAuditLogger:
|
||||
from litellm.proxy.proxy_server import create_config_audit_log
|
||||
|
||||
return create_config_audit_log
|
||||
|
||||
|
||||
async def _deployment_warning_reader(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> UserAPIKeyAuth:
|
||||
if user_api_key_dict.user_role not in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Only proxy admins and proxy admin viewers can read deployment warnings.",
|
||||
)
|
||||
return user_api_key_dict
|
||||
|
||||
|
||||
async def _deployment_warning_admin(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> UserAPIKeyAuth:
|
||||
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="Only proxy admins can ignore deployment warnings.",
|
||||
)
|
||||
return user_api_key_dict
|
||||
|
||||
|
||||
async def current_warning_conditions(now: Annotated[datetime, Depends(utc_now)]) -> tuple[WarningCondition, ...]:
|
||||
license_data: Final = read_license_data()
|
||||
return deployment_warning_conditions(
|
||||
env_credential_login_enabled=env_credential_login_enabled(),
|
||||
license_expiration_date=None if license_data is None else license_data.get("expiration_date"),
|
||||
no_redis_on_multiple_workers=await no_redis_on_multiple_workers(),
|
||||
no_redis_warning_disabled_by_env=no_redis_warning_disabled_by_env(),
|
||||
detailed_debug=verbose_logger.isEnabledFor(logging.DEBUG),
|
||||
now=now,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/health/deployment_warnings",
|
||||
tags=["health"],
|
||||
response_model=DeploymentWarningsResponse,
|
||||
)
|
||||
async def get_deployment_warnings(
|
||||
_: Annotated[UserAPIKeyAuth, Depends(_deployment_warning_reader)],
|
||||
conditions: Annotated[tuple[WarningCondition, ...], Depends(current_warning_conditions)],
|
||||
store: Annotated[DeploymentWarningIgnoreStore | None, Depends(deployment_warning_ignore_store)],
|
||||
now: Annotated[datetime, Depends(utc_now)],
|
||||
) -> DeploymentWarningsResponse:
|
||||
"""
|
||||
List the active deployment warnings with their severity and whether an admin ignored or snoozed them.
|
||||
Readable by proxy admins and proxy admin viewers.
|
||||
"""
|
||||
ignores: Final[Mapping[str, AdminIgnore]] = (
|
||||
MappingProxyType({}) if store is None else await _read_unexpired_ignores(store, now)
|
||||
)
|
||||
return DeploymentWarningsResponse(
|
||||
warnings=tuple(
|
||||
DeploymentWarning(id=condition.id, severity=condition.severity, ignore=_resolve_ignore(condition, ignores))
|
||||
for condition in conditions
|
||||
),
|
||||
ignore_available=store is not None,
|
||||
)
|
||||
|
||||
|
||||
@router.patch(
|
||||
"/health/deployment_warnings/{warning_id}",
|
||||
tags=["health"],
|
||||
response_model=DeploymentWarningIgnoreResponse,
|
||||
)
|
||||
async def update_deployment_warning_ignore(
|
||||
warning_id: str,
|
||||
update: DeploymentWarningIgnoreUpdate,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(_deployment_warning_admin)],
|
||||
store: Annotated[DeploymentWarningIgnoreStore | None, Depends(deployment_warning_ignore_store)],
|
||||
audit_logger: Annotated[ConfigAuditLogger, Depends(config_audit_logger)],
|
||||
now: Annotated[datetime, Depends(utc_now)],
|
||||
) -> DeploymentWarningIgnoreResponse:
|
||||
"""
|
||||
Ignore, snooze, or restore one deployment warning for every admin of this proxy.
|
||||
Only proxy admins can call this, and it needs a database.
|
||||
"""
|
||||
if not is_known_warning_id(warning_id):
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"Unknown deployment warning: {warning_id}")
|
||||
if store is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="Ignoring deployment warnings needs a database. Connect one to enable it.",
|
||||
)
|
||||
|
||||
before: Final = await _read_unexpired_ignores(store, now)
|
||||
ignore: Final = (
|
||||
AdminIgnore(
|
||||
ignored_by=user_api_key_dict.user_id or LITELLM_PROXY_ADMIN_NAME,
|
||||
ignored_at=now,
|
||||
ignored_until=None if update.snooze_seconds is None else now + timedelta(seconds=update.snooze_seconds),
|
||||
)
|
||||
if update.ignored
|
||||
else None
|
||||
)
|
||||
others: Final = {other_id: other for other_id, other in before.items() if other_id != warning_id}
|
||||
after: Final = MappingProxyType(others if ignore is None else {**others, warning_id: ignore})
|
||||
|
||||
await store.upsert_ignores(_IGNORES_ADAPTER.dump_json(dict(after)).decode())
|
||||
await audit_logger(
|
||||
param_name=DEPLOYMENT_WARNING_IGNORES_ROW_ID,
|
||||
action="updated",
|
||||
before_value=_ignores_as_audit_entries(before),
|
||||
after_value=_ignores_as_audit_entries(after),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
table_name=LitellmTableNames.UI_SETTINGS_TABLE_NAME,
|
||||
)
|
||||
return DeploymentWarningIgnoreResponse(id=warning_id, ignore=ignore)
|
||||
|
|
@ -590,6 +590,7 @@ from litellm.proxy.health_check import (
|
|||
perform_health_check,
|
||||
)
|
||||
from litellm.proxy.health_endpoints._health_endpoints import router as health_router
|
||||
from litellm.proxy.health_endpoints.deployment_warning_endpoints import router as deployment_warning_router
|
||||
from litellm.proxy.hooks.model_max_budget_limiter import ( # noqa: F401, RUF100 # legacy module exports
|
||||
PROXY_VirtualKeyModelMaxBudgetLimiter,
|
||||
_PROXY_VirtualKeyModelMaxBudgetLimiter, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
|
||||
|
|
@ -20402,6 +20403,7 @@ app.include_router(openai_files_router)
|
|||
reserve_lazy_slot(app, "llm_passthrough")
|
||||
app.include_router(pass_through_router)
|
||||
app.include_router(health_router)
|
||||
app.include_router(deployment_warning_router)
|
||||
app.include_router(key_management_router)
|
||||
app.include_router(internal_user_router)
|
||||
app.include_router(daily_activity_router)
|
||||
|
|
|
|||
23
litellm/repositories/deployment_warning_ignore_repository.py
Normal file
23
litellm/repositories/deployment_warning_ignore_repository.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.repositories.table_repositories import PrismaTableRepository
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from prisma import models as prisma_models # noqa: F401 # resolved only from the quoted base-class subscript below
|
||||
|
||||
DEPLOYMENT_WARNING_IGNORES_ROW_ID: Final = "deployment_warning_ignores"
|
||||
|
||||
|
||||
class DeploymentWarningIgnoreRepository(PrismaTableRepository["prisma_models.LiteLLM_UISettings"]):
|
||||
table_name = "litellm_uisettings"
|
||||
|
||||
async def get_raw_ignores(self) -> object:
|
||||
db_record: Final = await self.table.find_unique(where={"id": DEPLOYMENT_WARNING_IGNORES_ROW_ID})
|
||||
return db_record.ui_settings if db_record is not None else None
|
||||
|
||||
async def upsert_ignores(self, payload: str) -> None:
|
||||
row: Final = {"id": DEPLOYMENT_WARNING_IGNORES_ROW_ID, "ui_settings": payload}
|
||||
await self.table.upsert(
|
||||
where={"id": DEPLOYMENT_WARNING_IGNORES_ROW_ID},
|
||||
data={"create": row, "update": {"ui_settings": payload}},
|
||||
)
|
||||
|
|
@ -0,0 +1,475 @@
|
|||
import json
|
||||
import logging
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import LITELLM_PROXY_ADMIN_NAME
|
||||
from litellm.proxy._types import AUDIT_ACTIONS, LitellmTableNames, LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.health_endpoints.deployment_warning_endpoints import (
|
||||
WarningCondition,
|
||||
config_audit_logger,
|
||||
current_warning_conditions,
|
||||
deployment_warning_conditions,
|
||||
deployment_warning_ignore_store,
|
||||
license_expiry_condition,
|
||||
router,
|
||||
utc_now,
|
||||
)
|
||||
from litellm.proxy.proxy_server import (
|
||||
_dump_redacted_config, # pyright: ignore[reportPrivateUsage] # the exact redaction every config audit row goes through
|
||||
)
|
||||
from litellm.repositories.deployment_warning_ignore_repository import DEPLOYMENT_WARNING_IGNORES_ROW_ID
|
||||
|
||||
T0: Final = datetime(2026, 10, 14, 12, 0, tzinfo=timezone.utc)
|
||||
THIRTY_DAYS: Final = 30 * 24 * 60 * 60
|
||||
|
||||
|
||||
@dataclass
|
||||
class InMemoryIgnoreStore:
|
||||
raw: object = None
|
||||
|
||||
async def get_raw_ignores(self) -> object:
|
||||
return self.raw
|
||||
|
||||
async def upsert_ignores(self, payload: str) -> None:
|
||||
self.raw = json.loads(payload)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecordingAuditLogger:
|
||||
calls: list[dict[str, object]] = field(default_factory=list)
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
param_name: str,
|
||||
action: AUDIT_ACTIONS,
|
||||
before_value: JsonValue | None,
|
||||
after_value: JsonValue | None,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
table_name: LitellmTableNames = LitellmTableNames.CONFIG_TABLE_NAME,
|
||||
) -> None:
|
||||
self.calls.append(
|
||||
{
|
||||
"param_name": param_name,
|
||||
"action": action,
|
||||
"before_value": before_value,
|
||||
"after_value": after_value,
|
||||
"table_name": table_name,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Harness:
|
||||
app: FastAPI
|
||||
client: TestClient
|
||||
store: InMemoryIgnoreStore
|
||||
audit: RecordingAuditLogger
|
||||
clock: list[datetime]
|
||||
|
||||
def as_role(self, role: LitellmUserRoles, user_id: str | None = "admin-user") -> None:
|
||||
async def auth() -> UserAPIKeyAuth:
|
||||
return UserAPIKeyAuth(api_key="sk-test", user_id=user_id, user_role=role)
|
||||
|
||||
self.app.dependency_overrides[user_api_key_auth] = auth
|
||||
|
||||
def with_conditions(self, *conditions: WarningCondition) -> None:
|
||||
self.app.dependency_overrides[current_warning_conditions] = lambda: conditions
|
||||
|
||||
def without_database(self) -> None:
|
||||
self.app.dependency_overrides[deployment_warning_ignore_store] = lambda: None
|
||||
|
||||
def warnings(self) -> dict[str, dict[str, object]]:
|
||||
response = self.client.get("/health/deployment_warnings")
|
||||
assert response.status_code == 200, response.text
|
||||
return {warning["id"]: warning for warning in response.json()["warnings"]}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def harness() -> Iterator[Harness]:
|
||||
app = FastAPI()
|
||||
app.include_router(router)
|
||||
store = InMemoryIgnoreStore()
|
||||
audit = RecordingAuditLogger()
|
||||
clock = [T0]
|
||||
app.dependency_overrides[deployment_warning_ignore_store] = lambda: store
|
||||
app.dependency_overrides[config_audit_logger] = lambda: audit
|
||||
app.dependency_overrides[utc_now] = lambda: clock[0]
|
||||
h = Harness(app=app, client=TestClient(app), store=store, audit=audit, clock=clock)
|
||||
h.as_role(LitellmUserRoles.PROXY_ADMIN)
|
||||
h.with_conditions(
|
||||
WarningCondition(id="env_credential_login", severity="critical"),
|
||||
WarningCondition(id="detailed_debug", severity="warning"),
|
||||
)
|
||||
yield h
|
||||
|
||||
|
||||
class TestLicenseExpiryCondition:
|
||||
@pytest.mark.parametrize(
|
||||
("days_left", "severity"),
|
||||
[(31, None), (30, "warning"), (8, "warning"), (7, "critical"), (0, "critical"), (-1, "critical")],
|
||||
)
|
||||
def test_severity_follows_days_left(self, days_left, severity):
|
||||
expires_on = (T0 + timedelta(days=days_left)).date().isoformat()
|
||||
condition = license_expiry_condition(expires_on, T0)
|
||||
if severity is None:
|
||||
assert condition is None
|
||||
return
|
||||
assert condition == WarningCondition(id=f"license_expiry:{expires_on}", severity=severity)
|
||||
|
||||
def test_days_left_counts_utc_calendar_days(self):
|
||||
just_before_midnight = datetime(2026, 10, 14, 23, 59, tzinfo=timezone.utc)
|
||||
assert license_expiry_condition("2026-10-21", just_before_midnight).severity == "critical"
|
||||
assert license_expiry_condition("2026-10-22", just_before_midnight).severity == "warning"
|
||||
|
||||
def test_renewed_license_gets_a_new_id(self):
|
||||
first = license_expiry_condition("2026-10-20", T0)
|
||||
renewed = license_expiry_condition("2026-11-10", T0)
|
||||
assert first is not None and renewed is not None
|
||||
assert first.id != renewed.id
|
||||
|
||||
@pytest.mark.parametrize("expiration_date", [None, "", "not-a-date"])
|
||||
def test_missing_or_invalid_date_has_no_warning(self, expiration_date):
|
||||
assert license_expiry_condition(expiration_date, T0) is None
|
||||
|
||||
|
||||
class TestDeploymentWarningConditions:
|
||||
@staticmethod
|
||||
def _conditions(**overrides: object) -> dict[str, WarningCondition]:
|
||||
signals: dict[str, object] = {
|
||||
"env_credential_login_enabled": False,
|
||||
"license_expiration_date": None,
|
||||
"no_redis_on_multiple_workers": False,
|
||||
"no_redis_warning_disabled_by_env": False,
|
||||
"detailed_debug": False,
|
||||
"now": T0,
|
||||
**overrides,
|
||||
}
|
||||
return {c.id: c for c in deployment_warning_conditions(**signals)}
|
||||
|
||||
def test_healthy_deployment_has_no_warnings(self):
|
||||
assert self._conditions() == {}
|
||||
|
||||
def test_every_signal_maps_to_its_warning_and_severity(self):
|
||||
conditions = self._conditions(
|
||||
env_credential_login_enabled=True,
|
||||
license_expiration_date="2026-11-01",
|
||||
no_redis_on_multiple_workers=True,
|
||||
detailed_debug=True,
|
||||
)
|
||||
assert {cid: c.severity for cid, c in conditions.items()} == {
|
||||
"env_credential_login": "critical",
|
||||
"license_expiry:2026-11-01": "warning",
|
||||
"no_redis": "warning",
|
||||
"detailed_debug": "warning",
|
||||
}
|
||||
assert all(c.env_var_ignore is None for c in conditions.values())
|
||||
|
||||
def test_no_redis_env_var_keeps_the_warning_listed_as_env_ignored(self):
|
||||
conditions = self._conditions(no_redis_on_multiple_workers=True, no_redis_warning_disabled_by_env=True)
|
||||
assert conditions["no_redis"].env_var_ignore == "LITELLM_DISABLE_NO_REDIS_WARNING"
|
||||
|
||||
def test_env_var_alone_does_not_raise_the_no_redis_warning(self):
|
||||
assert self._conditions(no_redis_warning_disabled_by_env=True) == {}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def set_logger_level() -> Iterator[Callable[[int], None]]:
|
||||
original = verbose_logger.level
|
||||
yield verbose_logger.setLevel
|
||||
verbose_logger.setLevel(original)
|
||||
|
||||
|
||||
class TestCurrentWarningConditions:
|
||||
@staticmethod
|
||||
def _proxy_state(monkeypatch, *, redis_cache: object, general_settings: dict[str, object], license: object) -> None:
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.redis_usage_cache", redis_cache)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", general_settings)
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.premium_user_data", license)
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server._license_check",
|
||||
SimpleNamespace(airgapped_license_data=None, license_str=None, public_key=None),
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reads_every_signal_from_the_running_proxy(self, monkeypatch, set_logger_level):
|
||||
self._proxy_state(monkeypatch, redis_cache=None, general_settings={}, license={"expiration_date": "2026-10-20"})
|
||||
monkeypatch.setenv("LITELLM_DISABLE_NO_REDIS_WARNING", "true")
|
||||
set_logger_level(logging.DEBUG)
|
||||
|
||||
conditions = await current_warning_conditions(T0)
|
||||
|
||||
assert conditions == (
|
||||
WarningCondition(id="env_credential_login", severity="critical"),
|
||||
WarningCondition(id="license_expiry:2026-10-20", severity="critical"),
|
||||
WarningCondition(id="no_redis", severity="warning", env_var_ignore="LITELLM_DISABLE_NO_REDIS_WARNING"),
|
||||
WarningCondition(id="detailed_debug", severity="warning"),
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_healthy_proxy_has_no_conditions(self, monkeypatch, set_logger_level):
|
||||
self._proxy_state(
|
||||
monkeypatch,
|
||||
redis_cache=SimpleNamespace(),
|
||||
general_settings={"disable_env_credential_login": True},
|
||||
license=None,
|
||||
)
|
||||
monkeypatch.delenv("LITELLM_DISABLE_NO_REDIS_WARNING", raising=False)
|
||||
set_logger_level(logging.INFO)
|
||||
|
||||
assert await current_warning_conditions(T0) == ()
|
||||
|
||||
|
||||
class TestDeploymentWarningIgnoreStore:
|
||||
def test_no_database_means_no_store(self, monkeypatch):
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
|
||||
assert deployment_warning_ignore_store() is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reads_ignores_from_the_writer_not_a_lagging_replica(self, monkeypatch):
|
||||
def ui_settings_table(row: object) -> SimpleNamespace:
|
||||
async def find_unique(where: dict[str, str]) -> object:
|
||||
return row
|
||||
|
||||
return SimpleNamespace(litellm_uisettings=SimpleNamespace(find_unique=find_unique))
|
||||
|
||||
fresh = {"detailed_debug": {"ignored_by": "a", "ignored_at": "2026-10-14T12:00:00Z"}}
|
||||
monkeypatch.setattr(
|
||||
"litellm.proxy.proxy_server.prisma_client",
|
||||
SimpleNamespace(
|
||||
db=ui_settings_table(None), writer_db=ui_settings_table(SimpleNamespace(ui_settings=fresh))
|
||||
),
|
||||
)
|
||||
store = deployment_warning_ignore_store()
|
||||
assert store is not None
|
||||
assert await store.get_raw_ignores() == fresh
|
||||
|
||||
|
||||
class TestGetDeploymentWarnings:
|
||||
@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY])
|
||||
def test_admins_and_viewers_read_warnings(self, harness, role):
|
||||
harness.as_role(role)
|
||||
response = harness.client.get("/health/deployment_warnings")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"warnings": [
|
||||
{"id": "env_credential_login", "severity": "critical", "ignore": None},
|
||||
{"id": "detailed_debug", "severity": "warning", "ignore": None},
|
||||
],
|
||||
"ignore_available": True,
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"role", [LitellmUserRoles.INTERNAL_USER, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, LitellmUserRoles.TEAM]
|
||||
)
|
||||
def test_other_roles_are_rejected(self, harness, role):
|
||||
harness.as_role(role)
|
||||
assert harness.client.get("/health/deployment_warnings").status_code == 403
|
||||
|
||||
def test_env_var_ignore_is_reported(self, harness):
|
||||
harness.with_conditions(
|
||||
WarningCondition(id="no_redis", severity="warning", env_var_ignore="LITELLM_DISABLE_NO_REDIS_WARNING")
|
||||
)
|
||||
assert harness.warnings()["no_redis"]["ignore"] == {
|
||||
"source": "env_var",
|
||||
"env_var": "LITELLM_DISABLE_NO_REDIS_WARNING",
|
||||
}
|
||||
|
||||
def test_env_var_ignore_wins_over_a_stored_admin_ignore(self, harness):
|
||||
harness.with_conditions(WarningCondition(id="no_redis", severity="warning"))
|
||||
harness.client.patch("/health/deployment_warnings/no_redis", json={"ignored": True})
|
||||
harness.with_conditions(
|
||||
WarningCondition(id="no_redis", severity="warning", env_var_ignore="LITELLM_DISABLE_NO_REDIS_WARNING")
|
||||
)
|
||||
assert harness.warnings()["no_redis"]["ignore"]["source"] == "env_var"
|
||||
|
||||
def test_stored_ignore_without_timezone_reads_as_unreadable(self, harness):
|
||||
harness.store.raw = {
|
||||
"detailed_debug": {"ignored_by": "a", "ignored_at": "2026-10-14T12:00:00", "ignored_until": None}
|
||||
}
|
||||
warnings = harness.warnings()
|
||||
assert set(warnings) == {"env_credential_login", "detailed_debug"}
|
||||
assert warnings["detailed_debug"]["ignore"] is None
|
||||
|
||||
def test_corrupt_stored_ignores_read_as_none(self, harness):
|
||||
harness.store.raw = "{not json"
|
||||
warnings = harness.warnings()
|
||||
assert set(warnings) == {"env_credential_login", "detailed_debug"}
|
||||
assert all(w["ignore"] is None for w in warnings.values())
|
||||
|
||||
def test_without_database_ignoring_is_reported_unavailable(self, harness):
|
||||
harness.without_database()
|
||||
response = harness.client.get("/health/deployment_warnings")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["ignore_available"] is False
|
||||
assert [w["id"] for w in response.json()["warnings"]] == ["env_credential_login", "detailed_debug"]
|
||||
|
||||
|
||||
class TestUpdateDeploymentWarningIgnore:
|
||||
def test_snooze_computes_end_time_and_applies_to_every_admin(self, harness):
|
||||
harness.as_role(LitellmUserRoles.PROXY_ADMIN, user_id="alice")
|
||||
response = harness.client.patch(
|
||||
"/health/deployment_warnings/env_credential_login", json={"ignored": True, "snooze_seconds": 86400}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
expected_ignore = {
|
||||
"source": "admin",
|
||||
"ignored_by": "alice",
|
||||
"ignored_at": "2026-10-14T12:00:00Z",
|
||||
"ignored_until": "2026-10-15T12:00:00Z",
|
||||
}
|
||||
assert response.json() == {"id": "env_credential_login", "ignore": expected_ignore}
|
||||
|
||||
harness.as_role(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, user_id="bob")
|
||||
warnings = harness.warnings()
|
||||
assert warnings["env_credential_login"]["ignore"] == expected_ignore
|
||||
assert warnings["detailed_debug"]["ignore"] is None
|
||||
|
||||
def test_snooze_ends_exactly_at_its_end_time(self, harness):
|
||||
harness.client.patch("/health/deployment_warnings/detailed_debug", json={"ignored": True, "snooze_seconds": 60})
|
||||
|
||||
harness.clock[0] = T0 + timedelta(seconds=59)
|
||||
assert harness.warnings()["detailed_debug"]["ignore"] is not None
|
||||
|
||||
harness.clock[0] = T0 + timedelta(seconds=60)
|
||||
assert harness.warnings()["detailed_debug"]["ignore"] is None
|
||||
|
||||
def test_omitting_snooze_ignores_forever(self, harness):
|
||||
response = harness.client.patch("/health/deployment_warnings/detailed_debug", json={"ignored": True})
|
||||
assert response.status_code == 200
|
||||
assert response.json()["ignore"]["ignored_until"] is None
|
||||
|
||||
harness.clock[0] = T0 + timedelta(days=3650)
|
||||
assert harness.warnings()["detailed_debug"]["ignore"]["ignored_at"] == "2026-10-14T12:00:00Z"
|
||||
|
||||
def test_restore_clears_only_that_warning(self, harness):
|
||||
harness.client.patch("/health/deployment_warnings/detailed_debug", json={"ignored": True})
|
||||
harness.client.patch("/health/deployment_warnings/env_credential_login", json={"ignored": True})
|
||||
|
||||
response = harness.client.patch("/health/deployment_warnings/detailed_debug", json={"ignored": False})
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"id": "detailed_debug", "ignore": None}
|
||||
|
||||
warnings = harness.warnings()
|
||||
assert warnings["detailed_debug"]["ignore"] is None
|
||||
assert warnings["env_credential_login"]["ignore"]["source"] == "admin"
|
||||
|
||||
def test_any_severity_can_be_ignored(self, harness):
|
||||
harness.with_conditions(WarningCondition(id="license_expiry:2026-10-16", severity="critical"))
|
||||
response = harness.client.patch("/health/deployment_warnings/license_expiry:2026-10-16", json={"ignored": True})
|
||||
assert response.status_code == 200
|
||||
assert harness.warnings()["license_expiry:2026-10-16"]["ignore"]["ignored_by"] == "admin-user"
|
||||
|
||||
def test_ignoring_an_old_license_date_does_not_hide_the_renewed_one(self, harness):
|
||||
harness.client.patch("/health/deployment_warnings/license_expiry:2026-10-16", json={"ignored": True})
|
||||
harness.with_conditions(WarningCondition(id="license_expiry:2026-11-01", severity="warning"))
|
||||
assert harness.warnings()["license_expiry:2026-11-01"]["ignore"] is None
|
||||
|
||||
def test_change_is_written_to_the_config_audit_log(self, harness):
|
||||
harness.client.patch("/health/deployment_warnings/detailed_debug", json={"ignored": True})
|
||||
harness.client.patch("/health/deployment_warnings/detailed_debug", json={"ignored": False})
|
||||
|
||||
assert len(harness.audit.calls) == 2
|
||||
ignore_call, restore_call = harness.audit.calls
|
||||
assert ignore_call["table_name"] == LitellmTableNames.UI_SETTINGS_TABLE_NAME
|
||||
assert ignore_call["before_value"] == []
|
||||
assert ignore_call["after_value"] == [
|
||||
{
|
||||
"id": "detailed_debug",
|
||||
"ignore": {
|
||||
"source": "admin",
|
||||
"ignored_by": "admin-user",
|
||||
"ignored_at": "2026-10-14T12:00:00Z",
|
||||
"ignored_until": None,
|
||||
},
|
||||
}
|
||||
]
|
||||
assert restore_call["before_value"] == ignore_call["after_value"]
|
||||
assert restore_call["after_value"] == []
|
||||
assert {call["param_name"] for call in harness.audit.calls} == {DEPLOYMENT_WARNING_IGNORES_ROW_ID}
|
||||
assert {call["action"] for call in harness.audit.calls} == {"updated"}
|
||||
|
||||
def test_key_without_a_user_is_recorded_as_the_proxy_admin(self, harness):
|
||||
harness.as_role(LitellmUserRoles.PROXY_ADMIN, user_id=None)
|
||||
response = harness.client.patch("/health/deployment_warnings/detailed_debug", json={"ignored": True})
|
||||
assert response.json()["ignore"]["ignored_by"] == LITELLM_PROXY_ADMIN_NAME
|
||||
|
||||
def test_expired_snoozes_are_not_carried_into_the_next_write(self, harness):
|
||||
harness.client.patch("/health/deployment_warnings/detailed_debug", json={"ignored": True, "snooze_seconds": 60})
|
||||
harness.clock[0] = T0 + timedelta(minutes=5)
|
||||
harness.client.patch("/health/deployment_warnings/env_credential_login", json={"ignored": True})
|
||||
|
||||
assert [entry["id"] for entry in harness.audit.calls[-1]["after_value"]] == ["env_credential_login"]
|
||||
|
||||
def test_audit_entry_survives_secret_redaction(self, harness):
|
||||
harness.client.patch(
|
||||
"/health/deployment_warnings/env_credential_login", json={"ignored": True, "snooze_seconds": 3600}
|
||||
)
|
||||
|
||||
stored = json.loads(_dump_redacted_config(harness.audit.calls[-1]["after_value"]))
|
||||
|
||||
assert stored == [
|
||||
{
|
||||
"id": "env_credential_login",
|
||||
"ignore": {
|
||||
"source": "admin",
|
||||
"ignored_by": "admin-user",
|
||||
"ignored_at": "2026-10-14T12:00:00Z",
|
||||
"ignored_until": "2026-10-14T13:00:00Z",
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize("snooze_seconds", [0, -1, THIRTY_DAYS + 1])
|
||||
def test_snooze_outside_thirty_days_is_rejected(self, harness, snooze_seconds):
|
||||
response = harness.client.patch(
|
||||
"/health/deployment_warnings/detailed_debug", json={"ignored": True, "snooze_seconds": snooze_seconds}
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert harness.store.raw is None
|
||||
|
||||
def test_thirty_day_snooze_is_allowed(self, harness):
|
||||
response = harness.client.patch(
|
||||
"/health/deployment_warnings/detailed_debug", json={"ignored": True, "snooze_seconds": THIRTY_DAYS}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["ignore"]["ignored_until"] == "2026-11-13T12:00:00Z"
|
||||
|
||||
def test_snooze_without_ignoring_is_rejected(self, harness):
|
||||
response = harness.client.patch(
|
||||
"/health/deployment_warnings/detailed_debug", json={"ignored": False, "snooze_seconds": 60}
|
||||
)
|
||||
assert response.status_code == 422
|
||||
|
||||
@pytest.mark.parametrize("warning_id", ["made_up", "license_expiry:soon", "license_expiry:20261016"])
|
||||
def test_unknown_warning_is_rejected(self, harness, warning_id):
|
||||
response = harness.client.patch(f"/health/deployment_warnings/{warning_id}", json={"ignored": True})
|
||||
assert response.status_code == 404
|
||||
assert harness.store.raw is None
|
||||
|
||||
@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, LitellmUserRoles.INTERNAL_USER])
|
||||
def test_only_proxy_admins_can_ignore(self, harness, role):
|
||||
harness.as_role(role)
|
||||
response = harness.client.patch("/health/deployment_warnings/detailed_debug", json={"ignored": True})
|
||||
assert response.status_code == 403
|
||||
assert harness.store.raw is None
|
||||
assert harness.audit.calls == []
|
||||
|
||||
def test_without_database_ignoring_is_unavailable(self, harness):
|
||||
harness.without_database()
|
||||
response = harness.client.patch("/health/deployment_warnings/detailed_debug", json={"ignored": True})
|
||||
assert response.status_code == 503
|
||||
assert "database" in response.json()["detail"].lower()
|
||||
assert harness.audit.calls == []
|
||||
|
|
@ -4840,6 +4840,21 @@ class TestNoRedisWarning:
|
|||
assert await _show_no_redis_warning() is False
|
||||
prisma.db.query_raw.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("coordination_redis, router_redis", [(MagicMock(), None), (None, MagicMock())])
|
||||
async def test_redis_answers_before_the_env_var_is_looked_up(self, monkeypatch, coordination_redis, router_redis):
|
||||
monkeypatch.setattr(
|
||||
_health_endpoints_module,
|
||||
"get_secret_bool",
|
||||
MagicMock(side_effect=AssertionError("env var looked up although Redis is configured")),
|
||||
)
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.redis_usage_cache", coordination_redis),
|
||||
patch("litellm.proxy.proxy_server.llm_router", self._router(router_redis)),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", self._prisma_with_workers(5)),
|
||||
):
|
||||
assert await _show_no_redis_warning() is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stays_quiet_when_only_the_router_has_redis(self, monkeypatch):
|
||||
"""router_settings.redis_host alone backs cooldowns and usage-based routing."""
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.repositories.budget_repository import BudgetRepository
|
|||
from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE
|
||||
from litellm.repositories.config_repository import ConfigRepository
|
||||
from litellm.repositories.credentials_repository import CredentialsRepository
|
||||
from litellm.repositories.deployment_warning_ignore_repository import DeploymentWarningIgnoreRepository
|
||||
from litellm.repositories.model_repository import ModelRepository
|
||||
from litellm.repositories.object_permission_repository import (
|
||||
ObjectPermissionRepository,
|
||||
|
|
@ -1599,6 +1600,28 @@ class TestConfigRepository:
|
|||
assert len(params) == 2
|
||||
|
||||
|
||||
class TestDeploymentWarningIgnoreRepository:
|
||||
@pytest.mark.asyncio
|
||||
async def test_round_trips_on_the_writer_without_touching_other_ui_settings_rows(self):
|
||||
client = MockPrismaClient()
|
||||
writer_table = MockTable()
|
||||
client.writer_db = SimpleNamespace(litellm_uisettings=writer_table)
|
||||
other_rows = {
|
||||
"ui_settings": {"id": "ui_settings", "ui_settings": '{"disable_model_add_for_internal_users": true}'},
|
||||
"user_banner": {"id": "user_banner", "ui_settings": '{"enabled": false}'},
|
||||
}
|
||||
writer_table._records.update({row_id: dict(row) for row_id, row in other_rows.items()})
|
||||
repo = DeploymentWarningIgnoreRepository(client, use_writer=True)
|
||||
|
||||
assert await repo.get_raw_ignores() is None
|
||||
await repo.upsert_ignores('{"no_redis": {}}')
|
||||
await repo.upsert_ignores('{"detailed_debug": {}}')
|
||||
|
||||
assert await repo.get_raw_ignores() == '{"detailed_debug": {}}'
|
||||
assert {row_id: writer_table._records[row_id] for row_id in other_rows} == other_rows
|
||||
assert len(writer_table._records) == len(other_rows) + 1
|
||||
|
||||
|
||||
class TestVerificationTokenRepositoryExtended:
|
||||
@pytest.fixture
|
||||
def repo(self):
|
||||
|
|
|
|||
168
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
168
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -7502,6 +7502,48 @@ export interface paths {
|
|||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/health/deployment_warnings": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* Get Deployment Warnings
|
||||
* @description List the active deployment warnings with their severity and whether an admin ignored or snoozed them.
|
||||
* Readable by proxy admins and proxy admin viewers.
|
||||
*/
|
||||
get: operations["get_deployment_warnings_health_deployment_warnings_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/health/deployment_warnings/{warning_id}": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
/**
|
||||
* Update Deployment Warning Ignore
|
||||
* @description Ignore, snooze, or restore one deployment warning for every admin of this proxy.
|
||||
* Only proxy admins can call this, and it needs a database.
|
||||
*/
|
||||
patch: operations["update_deployment_warning_ignore_health_deployment_warnings__warning_id__patch"];
|
||||
trace?: never;
|
||||
};
|
||||
"/health/drain": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -25449,6 +25491,26 @@ export interface components {
|
|||
[key: string]: string;
|
||||
};
|
||||
};
|
||||
/** AdminIgnore */
|
||||
AdminIgnore: {
|
||||
/**
|
||||
* Ignored At
|
||||
* Format: date-time
|
||||
*/
|
||||
ignored_at: string;
|
||||
/** Ignored By */
|
||||
ignored_by: string;
|
||||
/**
|
||||
* Ignored Until
|
||||
* @description When the snooze ends. Null means the warning is ignored until an admin restores it.
|
||||
*/
|
||||
ignored_until?: string | null;
|
||||
/**
|
||||
* @description discriminator enum property added by openapi-typescript
|
||||
* @enum {string}
|
||||
*/
|
||||
source: "admin";
|
||||
};
|
||||
/**
|
||||
* AgentCapabilities
|
||||
* @description Defines optional capabilities supported by an agent.
|
||||
|
|
@ -31496,6 +31558,47 @@ export interface components {
|
|||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** DeploymentWarning */
|
||||
DeploymentWarning: {
|
||||
/** Id */
|
||||
id: string;
|
||||
/** Ignore */
|
||||
ignore?: (components["schemas"]["AdminIgnore"] | components["schemas"]["EnvVarIgnore"]) | null;
|
||||
/**
|
||||
* Severity
|
||||
* @enum {string}
|
||||
*/
|
||||
severity: "critical" | "warning";
|
||||
};
|
||||
/** DeploymentWarningIgnoreResponse */
|
||||
DeploymentWarningIgnoreResponse: {
|
||||
/** Id */
|
||||
id: string;
|
||||
ignore: components["schemas"]["AdminIgnore"] | null;
|
||||
};
|
||||
/** DeploymentWarningIgnoreUpdate */
|
||||
DeploymentWarningIgnoreUpdate: {
|
||||
/**
|
||||
* Ignored
|
||||
* @description True ignores the warning for every admin, false restores it.
|
||||
*/
|
||||
ignored: boolean;
|
||||
/**
|
||||
* Snooze Seconds
|
||||
* @description How long to ignore the warning, up to 30 days. Leave it out to ignore it forever.
|
||||
*/
|
||||
snooze_seconds?: number | null;
|
||||
};
|
||||
/** DeploymentWarningsResponse */
|
||||
DeploymentWarningsResponse: {
|
||||
/**
|
||||
* Ignore Available
|
||||
* @description False when no database is connected, so warnings cannot be ignored.
|
||||
*/
|
||||
ignore_available: boolean;
|
||||
/** Warnings */
|
||||
warnings: components["schemas"]["DeploymentWarning"][];
|
||||
};
|
||||
/** DiscoverAgentRequest */
|
||||
DiscoverAgentRequest: {
|
||||
/**
|
||||
|
|
@ -31834,6 +31937,16 @@ export interface components {
|
|||
/** Tenant Id */
|
||||
tenant_id: string;
|
||||
};
|
||||
/** EnvVarIgnore */
|
||||
EnvVarIgnore: {
|
||||
/** Env Var */
|
||||
env_var: string;
|
||||
/**
|
||||
* @description discriminator enum property added by openapi-typescript
|
||||
* @enum {string}
|
||||
*/
|
||||
source: "env_var";
|
||||
};
|
||||
/** EnvironmentReport */
|
||||
EnvironmentReport: {
|
||||
/** Config Lines */
|
||||
|
|
@ -61046,6 +61159,61 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
get_deployment_warnings_health_deployment_warnings_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["DeploymentWarningsResponse"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
update_deployment_warning_ignore_health_deployment_warnings__warning_id__patch: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path: {
|
||||
warning_id: string;
|
||||
};
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["DeploymentWarningIgnoreUpdate"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["DeploymentWarningIgnoreResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
health_drain_health_drain_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue