diff --git a/litellm/models/alert.py b/litellm/models/alert.py
index 4db127bf121..ae7463c92fc 100644
--- a/litellm/models/alert.py
+++ b/litellm/models/alert.py
@@ -5,10 +5,12 @@ from typing import Annotated, Final, Literal, Protocol, TypeAlias
from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, field_validator, model_validator
from typing_extensions import Self
-AlertType: TypeAlias = Literal["spend", "traffic", "latency", "reliability", "model_deprecation"]
+MetricAlertType: TypeAlias = Literal["spend", "traffic", "latency", "reliability"]
+AlertType: TypeAlias = MetricAlertType | Literal["model_deprecation", "prompt_cache"]
AlertFilter: TypeAlias = AlertType | Literal["performance"]
AlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"]
-AlertResourceType: TypeAlias = Literal["gateway", "key", "team", "user", "model"]
+AlertResourceType: TypeAlias = Literal["gateway", "key", "team", "user", "model", "deployment"]
+METRIC_ALERT_TYPES: Final[tuple[MetricAlertType, ...]] = ("spend", "traffic", "latency", "reliability")
MAX_ALERT_ID: Final = 2**63 - 1
@@ -41,6 +43,22 @@ class DeprecationEvidence(BaseModel):
)
+class PromptCacheEvidence(BaseModel):
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ config_fingerprint: str = Field(min_length=1, max_length=64, description="Cache-relevant deployment settings.")
+ prefix_fingerprint: str = Field(min_length=1, max_length=64, description="Keyed hash of the cached prefix.")
+ provider: Literal["anthropic"] = Field(description="Provider that served the requests.")
+ provider_model: str = Field(min_length=1, max_length=1024, description="Model the provider served.")
+ consecutive_misses: int = Field(
+ ge=1,
+ le=MAX_ALERT_ID,
+ description="Observed requests that reported no cache read or write, not necessarily consecutive.",
+ )
+ proven_write_tokens: int = Field(ge=1, le=MAX_ALERT_ID, description="Tokens the proving cache write stored.")
+ ttl_seconds: Literal[300] = Field(description="Cache lifetime of the prefix's breakpoint.")
+
+
class AlertSummary(BaseModel):
model_config = ConfigDict(frozen=True)
@@ -75,7 +93,9 @@ class AlertDetection(BaseModel):
window_end: AwareDatetime = Field(
description="Exclusive end of the observed window, normalized to UTC milliseconds."
)
- evidence: MetricEvidence | DeprecationEvidence = Field(description="Small saved snapshot explaining the alert.")
+ evidence: MetricEvidence | DeprecationEvidence | PromptCacheEvidence = Field(
+ description="Small saved snapshot explaining the alert."
+ )
@field_validator("window_start", "window_end")
@classmethod
@@ -96,6 +116,12 @@ class AlertDetection(BaseModel):
if not isinstance(self.evidence, DeprecationEvidence) or self.resource_type != "model":
raise ValueError("Model deprecations require a model resource and deprecation evidence")
return self
+ if self.type == "prompt_cache":
+ if not isinstance(self.evidence, PromptCacheEvidence) or self.resource_type != "deployment":
+ raise ValueError("Prompt cache alerts require a deployment resource and prompt cache evidence")
+ return self
+ if self.resource_type == "deployment":
+ raise ValueError("Only prompt cache alerts use a deployment resource")
expected_unit: Final = {
"spend": "usd",
"traffic": "requests",
@@ -109,6 +135,10 @@ class AlertDetection(BaseModel):
return self
def identity(self) -> tuple[str, ...]:
+ """One prompt cache incident per deployment and configuration, whatever window or prefix found it, so a
+ dismissed or resolved one is never reopened by a later detection."""
+ if isinstance(self.evidence, PromptCacheEvidence):
+ return (self.type, self.resource_type, self.resource_id, self.evidence.config_fingerprint)
return (
self.type,
self.resource_type,
diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py
index 6b878fa6afc..cddafa761ad 100644
--- a/litellm/repositories/alert_repository.py
+++ b/litellm/repositories/alert_repository.py
@@ -16,6 +16,7 @@ from typing_extensions import LiteralString
from litellm._logging import verbose_proxy_logger
from litellm.models.alert import (
+ METRIC_ALERT_TYPES,
Alert,
AlertDetection,
AlertFilter,
@@ -203,10 +204,11 @@ class AlertRepository:
LEFT JOIN "LiteLLM_VerificationToken" k ON a.resource_type = 'key' AND k.token = a.resource_id
LEFT JOIN "LiteLLM_TeamTable" t ON a.resource_type = 'team' AND t.team_id = a.resource_id
LEFT JOIN "LiteLLM_UserTable" u ON a.resource_type = 'user' AND u.user_id = a.resource_id
- WHERE a.id > $1 AND a.id <= $2 AND a.status = 'open' AND a.type != 'model_deprecation'
+ WHERE a.id > $1 AND a.id <= $2 AND a.status = 'open' AND a.type = ANY($3::text[])
ORDER BY a.id DESC LIMIT 5""",
after,
through,
+ list(METRIC_ALERT_TYPES),
)
return _SUMMARIES.validate_python(rows)
@@ -234,7 +236,8 @@ class AlertRepository:
self, status: AlertStatus, limit: int, cursor: int | None, alert_type: AlertFilter | None = None
) -> AlertPage:
size: Final = max(1, min(limit, 100))
- category: Final = {"not": "model_deprecation"} if alert_type == "performance" else alert_type
+ # bounded-ok: METRIC_ALERT_TYPES is the fixed four metric categories
+ category: Final = {"in": list(METRIC_ALERT_TYPES)} if alert_type == "performance" else alert_type
rows: Final = await self._table.find_many(
where={
"status": status,
@@ -269,11 +272,13 @@ class AlertRepository:
"start": (alert.window_start.date() - timedelta(days=7)).isoformat(),
}
for alert in alerts
- if alert.type != "model_deprecation"
+ if alert.type in METRIC_ALERT_TYPES
)
if not requested:
return tuple(AlertView.model_validate(alert.model_dump()) for alert in alerts)
- scopes: Final = frozenset[AlertResourceType](alert.resource_type for alert in alerts)
+ scopes: Final = frozenset[AlertResourceType](
+ alert.resource_type for alert in alerts if alert.type in METRIC_ALERT_TYPES
+ )
histories: Final = " UNION ALL ".join(_history_sql(scope) for scope in scopes)
query: Final = f"""WITH requested AS (
SELECT * FROM jsonb_to_recordset($1::jsonb)
diff --git a/litellm/repositories/performance_alert_repository.py b/litellm/repositories/performance_alert_repository.py
index 9eaaed77d3c..2f60c18d172 100644
--- a/litellm/repositories/performance_alert_repository.py
+++ b/litellm/repositories/performance_alert_repository.py
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # Prism
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import LiteralString
+from litellm.models.alert import METRIC_ALERT_TYPES
from litellm.models.performance_alert import (
BASELINE_DAYS,
FREE_ALERT_LIMIT,
@@ -178,9 +179,10 @@ class PerformanceAlertRepository:
await transaction.execute_raw(
"""UPDATE "LiteLLM_PerformanceAlertState" SET alerts_used = (
SELECT COUNT(*) FROM (SELECT 1 FROM "LiteLLM_Alert"
- WHERE type != 'model_deprecation' LIMIT $1) existing
+ WHERE type = ANY($2::text[]) LIMIT $1) existing
) WHERE id = 1 AND alerts_used IS NULL""",
FREE_ALERT_LIMIT,
+ list(METRIC_ALERT_TYPES),
)
return await self._monitoring(transaction, lock=lock)
return AlertMonitoring(
diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx
index 733641fdb17..679177de310 100644
--- a/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx
+++ b/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx
@@ -1,10 +1,11 @@
-import { ArrowLeft, KeyRound, Network, Server, User, Users } from "lucide-react";
+import { ArrowLeft, Cpu, KeyRound, Network, Server, User, Users } from "lucide-react";
import { useTranslation } from "react-i18next";
import { buttonVariants } from "@/components/ui/button";
import RecommendedReplacement from "@/components/ModelDeprecationAlerts/RecommendedReplacement";
import { migrationHref } from "@/components/ModelDeprecationAlerts/migrationRoute";
import AlertActions from "./AlertActions";
import AlertChart from "./AlertChart";
+import PromptCacheEvidence from "./PromptCacheEvidence";
import { AlertItem, dateLabel, presentAlert } from "./alertPresentation";
export default function AlertDetail({
@@ -18,7 +19,14 @@ export default function AlertDetail({
}) {
const { t } = useTranslation();
const alert = presentAlert(item);
- const Icon = { key: KeyRound, team: Users, user: User, model: Network, gateway: Server }[alert.resourceType];
+ const Icon = {
+ key: KeyRound,
+ team: Users,
+ user: User,
+ model: Network,
+ gateway: Server,
+ deployment: Cpu,
+ }[alert.resourceType];
return (