From e42e0db034ed83ecff0b6674d7482c1b22f958d1 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Mon, 16 Nov 2026 14:49:52 -0800 Subject: [PATCH] feat(alerts): display prompt cache advisories in existing alerts (#51943) * feat(alerts): read and show prompt cache alerts alongside the existing alert types Saved alerts can now carry a prompt_cache type on a deployment resource, with evidence made of opaque fingerprints and counters only. One incident exists per deployment and cache configuration, whatever window or prefix found it, so a dismissed or resolved one stays the only one. The performance list, the daily summary, the free-alert allowance and the history lookup now name the four metric types instead of excluding model deprecations, so these rows never reach metric charts, digests or the quota. The All view lists prompt cache alerts from their own query, leaving the performance query and its pagination unchanged. An alert's page shows the deployment, model, misses in a row, the proving write, the cache lifetime and when the misses happened, in place of a metric chart, and keeps the existing dismiss, resolve and reopen controls. * fix(alerts): pin prompt cache evidence to what is observed and prove the readers against the database Prompt cache evidence now only accepts the anthropic provider and the five-minute TTL. The title says cache usage was not observed rather than that caching stopped, and the scope copy names the checked requests: non-streaming Anthropic Chat Completions with one five-minute breakpoint, after a confirmed write, with never-proven prefixes left unassessed. Alerts are presented by their evidence shape. Metric labels come from the evidence unit and the activity link label skips deployments, so the unrecognized-alert fallback is gone. The real-database alert tests now round-trip a prompt cache alert, keep it out of the performance list and the daily summary, keep a dismissed one as the only incident, and leave it out of the free allowance. They replace the unit tests that only checked SQL arguments. The UI tests reopen an incident, page both lists in All from their own cursors, and drop loaded prompt cache alerts when Performance is chosen. * chore(ui): regenerate API types for prompt cache alerts Alerts can now carry the prompt_cache type, the deployment resource and prompt cache evidence, so the generated dashboard types include them. * fix(alerts): satisfy the dashboard lint and the bounded IN check for prompt cache alerts The prompt cache list query reuses useAlerts with the prompt_cache type, so no five-parameter helper is needed. The new prompt cache mock handling in the Alerts integration test lives in its own function, a presentation test names its expected details, and AlertDetail is formatted. The performance list's IN filter over the four fixed metric types records that bound. * fix(alerts): reject metric alerts on a deployment resource Only prompt cache alerts use the new deployment resource. Metric alerts have no deployment history source, so a metric alert on a deployment is now invalid. Every combination accepted before this change is still accepted. * test(alerts): match the plain token count the prompt cache evidence renders * fix(ui): describe prompt cache alerts as advisory observed evidence The alert reports requests the proxy observed. Delayed, missing or unlogged requests can fall between them, so the list and detail say they are not necessarily consecutive and do not prove an outage. The serialized consecutive_misses field is unchanged. * fix(ui): title prompt cache alerts as advisories and attribute the limit to observations The title no longer reads as a deployment-wide finding, and the limitation says delayed, missing or unlogged observations mean the requests are not necessarily consecutive and do not prove a caching outage. * docs(alerts): describe consecutive_misses as observed requests that are not necessarily consecutive * chore(ui): regenerate API types for the advisory prompt cache evidence description --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/models/alert.py | 36 ++++- litellm/repositories/alert_repository.py | 13 +- .../performance_alert_repository.py | 4 +- .../src/components/Alerts/AlertDetail.tsx | 14 +- .../Alerts/Alerts.integration.test.tsx | 132 +++++++++++++++++- .../src/components/Alerts/Alerts.tsx | 39 ++++-- .../components/Alerts/PromptCacheEvidence.tsx | 44 ++++++ .../Alerts/alertPresentation.test.ts | 42 ++++++ .../components/Alerts/alertPresentation.ts | 67 ++++++++- .../src/components/Alerts/useAlerts.ts | 10 +- .../src/i18n/locales/en.json | 20 ++- .../src/lib/http/schema.d.ts | 56 ++++++-- tests/integration/database/test_alerts.py | 54 ++++++- .../database/test_performance_alerts.py | 24 +++- tests/unit/models/test_alert.py | 56 ++++++++ .../repositories/test_alert_repository.py | 104 ++++++++++++++ .../test_performance_alert_repository.py | 2 +- 17 files changed, 667 insertions(+), 50 deletions(-) create mode 100644 litellm/ui/litellm-dashboard/src/components/Alerts/PromptCacheEvidence.tsx create mode 100644 tests/unit/repositories/test_alert_repository.py diff --git a/litellm/models/alert.py b/litellm/models/alert.py index 4db127bf121..ae7463c92fc 100644 --- a/litellm/models/alert.py +++ b/litellm/models/alert.py @@ -5,10 +5,12 @@ from typing import Annotated, Final, Literal, Protocol, TypeAlias from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, field_validator, model_validator from typing_extensions import Self -AlertType: TypeAlias = Literal["spend", "traffic", "latency", "reliability", "model_deprecation"] +MetricAlertType: TypeAlias = Literal["spend", "traffic", "latency", "reliability"] +AlertType: TypeAlias = MetricAlertType | Literal["model_deprecation", "prompt_cache"] AlertFilter: TypeAlias = AlertType | Literal["performance"] AlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] -AlertResourceType: TypeAlias = Literal["gateway", "key", "team", "user", "model"] +AlertResourceType: TypeAlias = Literal["gateway", "key", "team", "user", "model", "deployment"] +METRIC_ALERT_TYPES: Final[tuple[MetricAlertType, ...]] = ("spend", "traffic", "latency", "reliability") MAX_ALERT_ID: Final = 2**63 - 1 @@ -41,6 +43,22 @@ class DeprecationEvidence(BaseModel): ) +class PromptCacheEvidence(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + config_fingerprint: str = Field(min_length=1, max_length=64, description="Cache-relevant deployment settings.") + prefix_fingerprint: str = Field(min_length=1, max_length=64, description="Keyed hash of the cached prefix.") + provider: Literal["anthropic"] = Field(description="Provider that served the requests.") + provider_model: str = Field(min_length=1, max_length=1024, description="Model the provider served.") + consecutive_misses: int = Field( + ge=1, + le=MAX_ALERT_ID, + description="Observed requests that reported no cache read or write, not necessarily consecutive.", + ) + proven_write_tokens: int = Field(ge=1, le=MAX_ALERT_ID, description="Tokens the proving cache write stored.") + ttl_seconds: Literal[300] = Field(description="Cache lifetime of the prefix's breakpoint.") + + class AlertSummary(BaseModel): model_config = ConfigDict(frozen=True) @@ -75,7 +93,9 @@ class AlertDetection(BaseModel): window_end: AwareDatetime = Field( description="Exclusive end of the observed window, normalized to UTC milliseconds." ) - evidence: MetricEvidence | DeprecationEvidence = Field(description="Small saved snapshot explaining the alert.") + evidence: MetricEvidence | DeprecationEvidence | PromptCacheEvidence = Field( + description="Small saved snapshot explaining the alert." + ) @field_validator("window_start", "window_end") @classmethod @@ -96,6 +116,12 @@ class AlertDetection(BaseModel): if not isinstance(self.evidence, DeprecationEvidence) or self.resource_type != "model": raise ValueError("Model deprecations require a model resource and deprecation evidence") return self + if self.type == "prompt_cache": + if not isinstance(self.evidence, PromptCacheEvidence) or self.resource_type != "deployment": + raise ValueError("Prompt cache alerts require a deployment resource and prompt cache evidence") + return self + if self.resource_type == "deployment": + raise ValueError("Only prompt cache alerts use a deployment resource") expected_unit: Final = { "spend": "usd", "traffic": "requests", @@ -109,6 +135,10 @@ class AlertDetection(BaseModel): return self def identity(self) -> tuple[str, ...]: + """One prompt cache incident per deployment and configuration, whatever window or prefix found it, so a + dismissed or resolved one is never reopened by a later detection.""" + if isinstance(self.evidence, PromptCacheEvidence): + return (self.type, self.resource_type, self.resource_id, self.evidence.config_fingerprint) return ( self.type, self.resource_type, diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py index 6b878fa6afc..cddafa761ad 100644 --- a/litellm/repositories/alert_repository.py +++ b/litellm/repositories/alert_repository.py @@ -16,6 +16,7 @@ from typing_extensions import LiteralString from litellm._logging import verbose_proxy_logger from litellm.models.alert import ( + METRIC_ALERT_TYPES, Alert, AlertDetection, AlertFilter, @@ -203,10 +204,11 @@ class AlertRepository: LEFT JOIN "LiteLLM_VerificationToken" k ON a.resource_type = 'key' AND k.token = a.resource_id LEFT JOIN "LiteLLM_TeamTable" t ON a.resource_type = 'team' AND t.team_id = a.resource_id LEFT JOIN "LiteLLM_UserTable" u ON a.resource_type = 'user' AND u.user_id = a.resource_id - WHERE a.id > $1 AND a.id <= $2 AND a.status = 'open' AND a.type != 'model_deprecation' + WHERE a.id > $1 AND a.id <= $2 AND a.status = 'open' AND a.type = ANY($3::text[]) ORDER BY a.id DESC LIMIT 5""", after, through, + list(METRIC_ALERT_TYPES), ) return _SUMMARIES.validate_python(rows) @@ -234,7 +236,8 @@ class AlertRepository: self, status: AlertStatus, limit: int, cursor: int | None, alert_type: AlertFilter | None = None ) -> AlertPage: size: Final = max(1, min(limit, 100)) - category: Final = {"not": "model_deprecation"} if alert_type == "performance" else alert_type + # bounded-ok: METRIC_ALERT_TYPES is the fixed four metric categories + category: Final = {"in": list(METRIC_ALERT_TYPES)} if alert_type == "performance" else alert_type rows: Final = await self._table.find_many( where={ "status": status, @@ -269,11 +272,13 @@ class AlertRepository: "start": (alert.window_start.date() - timedelta(days=7)).isoformat(), } for alert in alerts - if alert.type != "model_deprecation" + if alert.type in METRIC_ALERT_TYPES ) if not requested: return tuple(AlertView.model_validate(alert.model_dump()) for alert in alerts) - scopes: Final = frozenset[AlertResourceType](alert.resource_type for alert in alerts) + scopes: Final = frozenset[AlertResourceType]( + alert.resource_type for alert in alerts if alert.type in METRIC_ALERT_TYPES + ) histories: Final = " UNION ALL ".join(_history_sql(scope) for scope in scopes) query: Final = f"""WITH requested AS ( SELECT * FROM jsonb_to_recordset($1::jsonb) diff --git a/litellm/repositories/performance_alert_repository.py b/litellm/repositories/performance_alert_repository.py index 9eaaed77d3c..2f60c18d172 100644 --- a/litellm/repositories/performance_alert_repository.py +++ b/litellm/repositories/performance_alert_repository.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # Prism from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import LiteralString +from litellm.models.alert import METRIC_ALERT_TYPES from litellm.models.performance_alert import ( BASELINE_DAYS, FREE_ALERT_LIMIT, @@ -178,9 +179,10 @@ class PerformanceAlertRepository: await transaction.execute_raw( """UPDATE "LiteLLM_PerformanceAlertState" SET alerts_used = ( SELECT COUNT(*) FROM (SELECT 1 FROM "LiteLLM_Alert" - WHERE type != 'model_deprecation' LIMIT $1) existing + WHERE type = ANY($2::text[]) LIMIT $1) existing ) WHERE id = 1 AND alerts_used IS NULL""", FREE_ALERT_LIMIT, + list(METRIC_ALERT_TYPES), ) return await self._monitoring(transaction, lock=lock) return AlertMonitoring( diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx index 733641fdb17..679177de310 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx @@ -1,10 +1,11 @@ -import { ArrowLeft, KeyRound, Network, Server, User, Users } from "lucide-react"; +import { ArrowLeft, Cpu, KeyRound, Network, Server, User, Users } from "lucide-react"; import { useTranslation } from "react-i18next"; import { buttonVariants } from "@/components/ui/button"; import RecommendedReplacement from "@/components/ModelDeprecationAlerts/RecommendedReplacement"; import { migrationHref } from "@/components/ModelDeprecationAlerts/migrationRoute"; import AlertActions from "./AlertActions"; import AlertChart from "./AlertChart"; +import PromptCacheEvidence from "./PromptCacheEvidence"; import { AlertItem, dateLabel, presentAlert } from "./alertPresentation"; export default function AlertDetail({ @@ -18,7 +19,14 @@ export default function AlertDetail({ }) { const { t } = useTranslation(); const alert = presentAlert(item); - const Icon = { key: KeyRound, team: Users, user: User, model: Network, gateway: Server }[alert.resourceType]; + const Icon = { + key: KeyRound, + team: Users, + user: User, + model: Network, + gateway: Server, + deployment: Cpu, + }[alert.resourceType]; return (
@@ -158,6 +166,9 @@ function AlertsInbox({ auth }: { auth: ReturnType }) { {showDeprecations && deprecations.isError && ( void deprecations.refetch()}>{t("alerts.unified.deprecationsError")} )} + {showPromptCache && promptCache.isError && ( + void promptCache.refetch()}>{t("alerts.unified.promptCacheError")} + )} {showPerformance && list.isError && ( void list.refetch()}>{t("alerts.unified.performanceError")} )} @@ -194,14 +205,16 @@ function AlertsInbox({ auth }: { auth: ReturnType }) { ); })} - {showPerformance && list.hasNextPage && ( + {pagedLists.some((query) => query.hasNextPage) && ( )} diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/PromptCacheEvidence.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/PromptCacheEvidence.tsx new file mode 100644 index 00000000000..1c0091f9db8 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/PromptCacheEvidence.tsx @@ -0,0 +1,44 @@ +import { useTranslation } from "react-i18next"; +import type { PromptCacheDetails } from "./alertPresentation"; + +const timeFormat = { dateStyle: "medium", timeStyle: "medium", timeZone: "UTC" } as const; + +export default function PromptCacheEvidence({ details }: { details: PromptCacheDetails }) { + const { t } = useTranslation(); + const facts: [string, string][] = [ + [t("alerts.unified.promptCache.deployment"), details.deploymentId], + [t("alerts.unified.promptCache.model"), details.model], + [t("alerts.unified.promptCache.misses"), details.misses.toLocaleString()], + [ + t("alerts.unified.promptCache.provenWrite"), + t("alerts.unified.promptCache.provenWriteTokens", { count: details.provenWriteTokens }), + ], + [ + t("alerts.unified.promptCache.ttl"), + t("alerts.unified.promptCache.ttlMinutes", { count: details.ttlSeconds / 60 }), + ], + [ + t("alerts.unified.promptCache.window"), + t("alerts.unified.promptCache.windowValue", { + start: new Date(details.firstMiss).toLocaleString(undefined, timeFormat), + end: new Date(details.lastMiss).toLocaleString(undefined, timeFormat), + }), + ], + ]; + return ( +
+

+ {t("alerts.unified.promptCache.evidenceHeading")} +

+
+ {facts.map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+

{t("alerts.unified.promptCache.scope")}

+
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts b/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts index aa92ca8fc20..47a473260a4 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts @@ -31,6 +31,48 @@ describe("alert presentation", () => { expect(presentation.description).toContain(formatted); expect(presentation.baseline).toBe(baseline); }); + it("presents a prompt cache incident without reading it as a metric or deprecation alert", () => { + const presentation = presentAlert({ + source: "performance", + alert: { + ...saved, + type: "prompt_cache", + resource_type: "deployment", + resource_id: "claude-east", + evidence: { + config_fingerprint: "config-1", + prefix_fingerprint: "prefix-1", + provider: "anthropic", + provider_model: "claude-test", + consecutive_misses: 3, + proven_write_tokens: 2048, + ttl_seconds: 300, + }, + }, + }); + expect(presentation.title).toBe("Prompt cache usage advisory for claude-east"); + expect(presentation.description).toContain("3 observed requests to claude-test reported no cache usage"); + expect(presentation.description).toContain("not necessarily consecutive and do not prove a caching outage"); + expect(presentation.description).not.toContain("prefix-1"); + expect([presentation.resourceType, presentation.href, presentation.history]).toEqual([ + "deployment", + undefined, + null, + ]); + const expectedDetails = { + deploymentId: "claude-east", + model: "claude-test", + misses: 3, + provenWriteTokens: 2048, + ttlSeconds: 300, + firstMiss: saved.window_start, + lastMiss: saved.window_end, + }; + expect(presentation.promptCache).toEqual(expectedDetails); + }); + it("keeps metric alerts on their chart", () => { + expect(presentAlert({ source: "performance", alert: saved }).promptCache).toBeNull(); + }); it("links to the resource with an inclusive eight-day window", () => { expect(activityLink(saved)).toBe( "/ui/teams?team=example-team&detail_tab=activity&activity_from=2026-01-01&activity_to=2026-01-08", diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts b/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts index 79a5d84a2a4..ef408722236 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts @@ -1,4 +1,4 @@ -import type { components } from "@/lib/http/schema"; +import type { components, operations } from "@/lib/http/schema"; import i18n from "@/i18n/i18n"; import type { DeprecationAlert } from "@/components/ModelDeprecationAlerts/deprecationAlerts"; import { effectiveAlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; @@ -11,6 +11,15 @@ export type AlertItem = | { source: "deprecation"; alert: DeprecationAlert }; export type MetricUnit = "usd" | "requests" | "milliseconds" | "ratio"; export type HistoryPoint = { date: string; value: number | null }; +export type PromptCacheDetails = { + deploymentId: string; + model: string; + misses: number; + provenWriteTokens: number; + ttlSeconds: number; + firstMiss: string; + lastMiss: string; +}; export const ALERT_TYPES = [ "all", "performance", @@ -21,6 +30,23 @@ export const ALERT_TYPES = [ "reliability", ] as const; export type AlertFilter = (typeof ALERT_TYPES)[number]; +export type ListedAlertType = NonNullable< + NonNullable["type"] +>; +export function listedAlertType(filter: AlertFilter): ListedAlertType | null { + if (filter === "model-deprecations") return null; + return filter === "all" ? "performance" : filter; +} +const METRIC_LABEL_KEYS = { + usd: "alerts.unified.metrics.spend", + requests: "alerts.unified.metrics.traffic", + milliseconds: "alerts.unified.metrics.latency", + ratio: "alerts.unified.metrics.reliability", +} as const; +function linkLabel(resource: SavedAlert["resource_type"]): string { + if (resource === "deployment") return ""; + return i18n.t(`alerts.unified.links.${resource}`); +} export const TYPE_LABEL_KEYS = { all: "alerts.unified.types.all", performance: "alerts.unified.types.performance", @@ -98,6 +124,7 @@ export function presentAlert(item: AlertItem) { [i18n.t("alerts.unified.lastUsed"), alert.lastUsedDate], ], chartTitle: i18n.t("alerts.usageChart.title"), + promptCache: null, }; } const alert = item.alert; @@ -106,6 +133,38 @@ export function presentAlert(item: AlertItem) { (alert.resource_type === "key" ? i18n.t("alerts.unified.keyFallback", { id: alert.resource_id.slice(0, 8) }) : alert.resource_id); + if ("consecutive_misses" in alert.evidence) { + const evidence = alert.evidence; + const promptCache: PromptCacheDetails = { + deploymentId: alert.resource_id, + model: evidence.provider_model, + misses: evidence.consecutive_misses, + provenWriteTokens: evidence.proven_write_tokens, + ttlSeconds: evidence.ttl_seconds, + firstMiss: alert.window_start, + lastMiss: alert.window_end, + }; + return { + id: alert.id, + title: i18n.t("alerts.unified.promptCache.title", { name }), + description: i18n.t("alerts.unified.promptCache.description", { + misses: evidence.consecutive_misses.toLocaleString(), + model: evidence.provider_model, + }), + status: alert.status, + resourceType: alert.resource_type, + resourceName: name, + href: undefined, + linkLabel: "", + unit: "requests" as MetricUnit, + history: null, + baseline: undefined, + observed: alert.window_start.slice(0, 10), + dates: [[i18n.t("alerts.unified.firstDetected"), alert.created_at]], + chartTitle: i18n.t("alerts.usageChart.title"), + promptCache, + }; + } if (!("value" in alert.evidence)) { return { id: alert.id, @@ -122,11 +181,12 @@ export function presentAlert(item: AlertItem) { observed: alert.window_start.slice(0, 10), dates: [[i18n.t("alerts.unified.retires"), alert.evidence.deprecation_date]], chartTitle: i18n.t("alerts.usageChart.title"), + promptCache: null, }; } const evidence = alert.evidence; const observed = alert.window_start.slice(0, 10); - const label = i18n.t(`alerts.unified.metrics.${alert.type}`); + const label = i18n.t(METRIC_LABEL_KEYS[evidence.unit]); const direction = evidence.value < evidence.baseline ? "fell" : "rose"; return { id: alert.id, @@ -151,7 +211,7 @@ export function presentAlert(item: AlertItem) { resourceType: alert.resource_type, resourceName: name, href: activityLink(alert), - linkLabel: i18n.t(`alerts.unified.links.${alert.resource_type}`), + linkLabel: linkLabel(alert.resource_type), unit: evidence.unit, history: alert.history, baseline: evidence.baseline, @@ -161,6 +221,7 @@ export function presentAlert(item: AlertItem) { [i18n.t("alerts.unified.firstDetected"), alert.created_at], ], chartTitle: i18n.t(`alerts.unified.chart.titles.${evidence.unit}`), + promptCache: null, }; } export type AlertPresentation = ReturnType; diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts b/litellm/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts index 70813ef2e93..ea79bd26460 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts @@ -3,17 +3,17 @@ import { useInfiniteQuery, useQuery } from "@tanstack/react-query"; import { backgroundFetchClient } from "@/lib/http/api"; import { getProxyBaseUrl } from "@/components/networking"; import { AlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; -import type { AlertFilter } from "./alertPresentation"; +import { listedAlertType, type AlertFilter } from "./alertPresentation"; -export function useAlerts(scope: string, enabled: boolean, status: AlertStatus, filter: AlertFilter) { +export function useAlerts(scope: string, enabled: boolean, status: AlertStatus, filter: AlertFilter | "prompt_cache") { + const type = filter === "prompt_cache" ? filter : listedAlertType(filter); const options = { queryKey: ["alerts", getProxyBaseUrl(), scope, status, filter], - enabled: enabled && filter !== "model-deprecations", + enabled: enabled && type !== null, initialPageParam: undefined as string | undefined, queryFn: async ({ pageParam, signal }: { pageParam: string | undefined; signal: AbortSignal }) => { - const type = filter === "all" || filter === "model-deprecations" ? "performance" : filter; const { data } = await backgroundFetchClient.GET("/alerts", { - params: { query: { status, type, limit: 25, cursor: pageParam } }, + params: { query: { status, type: type ?? undefined, limit: 25, cursor: pageParam } }, signal, }); if (!data) throw new Error("Could not load alerts"); diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json index e25275d0612..8f755b63373 100644 --- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json @@ -5868,6 +5868,23 @@ }, "retiredTitle": "{{name}} retired on {{date}}", "retiresTitle": "{{name}} retires on {{date}}", + "promptCache": { + "title": "Prompt cache usage advisory for {{name}}", + "description": "{{misses}} observed requests to {{model}} reported no cache usage for a prefix the provider had cached before. Delayed, missing or unlogged observations mean these requests are not necessarily consecutive and do not prove a caching outage.", + "evidenceHeading": "What was observed", + "deployment": "Deployment", + "model": "Model", + "misses": "Observed requests with no cache usage", + "provenWrite": "Earlier cache write that proved the prefix", + "provenWriteTokens_one": "{{count}} token", + "provenWriteTokens_other": "{{count}} tokens", + "ttl": "Cache lifetime", + "ttlMinutes_one": "{{count}} minute", + "ttlMinutes_other": "{{count}} minutes", + "window": "Misses observed", + "windowValue": "{{start}} to {{end}} UTC", + "scope": "Only non-streaming Anthropic Chat Completions requests with one five-minute cache breakpoint are checked, after the provider confirmed a cache write for that prefix. Prefixes never confirmed as cached are not assessed." + }, "deprecationDescription": "{{requests}} recorded requests. Last used {{date}} UTC.", "retired": "Retired", "retires": "Retires", @@ -5913,7 +5930,8 @@ "milliseconds": "Mean response time per day", "ratio": "Daily error rate" } - } + }, + "promptCacheError": "Could not load prompt cache alerts." }, "page": { "tabs": { diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index 681a91b3809..ada43316510 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27698,7 +27698,7 @@ export interface components { * Evidence * @description Small saved snapshot explaining the alert. */ - evidence: components["schemas"]["MetricEvidence"] | components["schemas"]["DeprecationEvidence"]; + evidence: components["schemas"]["MetricEvidence"] | components["schemas"]["DeprecationEvidence"] | components["schemas"]["PromptCacheEvidence"]; /** * Id * @description Opaque alert ID, also used as a pagination cursor. @@ -27714,7 +27714,7 @@ export interface components { * @description Kind of affected resource. * @enum {string} */ - resource_type: "gateway" | "key" | "team" | "user" | "model"; + resource_type: "gateway" | "key" | "team" | "user" | "model" | "deployment"; /** * Status * @description Current review status. @@ -27724,9 +27724,8 @@ export interface components { /** * Type * @description Built-in alert category. - * @enum {string} */ - type: "spend" | "traffic" | "latency" | "reliability" | "model_deprecation"; + type: ("spend" | "traffic" | "latency" | "reliability") | ("model_deprecation" | "prompt_cache"); /** * Updated At * Format: date-time @@ -27829,7 +27828,7 @@ export interface components { * Evidence * @description Small saved snapshot explaining the alert. */ - evidence: components["schemas"]["MetricEvidence"] | components["schemas"]["DeprecationEvidence"]; + evidence: components["schemas"]["MetricEvidence"] | components["schemas"]["DeprecationEvidence"] | components["schemas"]["PromptCacheEvidence"]; /** * History * @description Current Usage values for the seven baseline days and observed day. Null if unavailable. @@ -27855,7 +27854,7 @@ export interface components { * @description Kind of affected resource. * @enum {string} */ - resource_type: "gateway" | "key" | "team" | "user" | "model"; + resource_type: "gateway" | "key" | "team" | "user" | "model" | "deployment"; /** * Status * @description Current review status. @@ -27865,9 +27864,8 @@ export interface components { /** * Type * @description Built-in alert category. - * @enum {string} */ - type: "spend" | "traffic" | "latency" | "reliability" | "model_deprecation"; + type: ("spend" | "traffic" | "latency" | "reliability") | ("model_deprecation" | "prompt_cache"); /** * Updated At * Format: date-time @@ -44680,6 +44678,46 @@ export interface components { prompt_id: string; prompt_info?: components["schemas"]["PromptInfo"] | null; }; + /** PromptCacheEvidence */ + PromptCacheEvidence: { + /** + * Config Fingerprint + * @description Cache-relevant deployment settings. + */ + config_fingerprint: string; + /** + * Consecutive Misses + * @description Observed requests that reported no cache read or write, not necessarily consecutive. + */ + consecutive_misses: number; + /** + * Prefix Fingerprint + * @description Keyed hash of the cached prefix. + */ + prefix_fingerprint: string; + /** + * Proven Write Tokens + * @description Tokens the proving cache write stored. + */ + proven_write_tokens: number; + /** + * Provider + * @description Provider that served the requests. + * @constant + */ + provider: "anthropic"; + /** + * Provider Model + * @description Model the provider served. + */ + provider_model: string; + /** + * Ttl Seconds + * @description Cache lifetime of the prefix's breakpoint. + * @constant + */ + ttl_seconds: 300; + }; /** PromptCacheOptions */ PromptCacheOptions: { /** Comparison Response Id */ @@ -56290,7 +56328,7 @@ export interface operations { /** @description next_cursor from the preceding page. */ cursor?: string | null; /** @description Alert category, or all performance alerts. */ - type?: ("spend" | "traffic" | "latency" | "reliability" | "model_deprecation") | "performance" | null; + type?: ("spend" | "traffic" | "latency" | "reliability") | ("model_deprecation" | "prompt_cache") | "performance" | null; }; header?: never; path?: never; diff --git a/tests/integration/database/test_alerts.py b/tests/integration/database/test_alerts.py index 06908c3a569..e15a24133a6 100644 --- a/tests/integration/database/test_alerts.py +++ b/tests/integration/database/test_alerts.py @@ -28,6 +28,16 @@ from tests.integration._support.client import Gateway from tests.integration._support.database import read_rows, write_rows +PROMPT_CACHE_EVIDENCE: Final = { + "config_fingerprint": "config-1", + "provider": "anthropic", + "provider_model": "claude-sonnet-5-5", + "consecutive_misses": 3, + "proven_write_tokens": 2048, + "ttl_seconds": 300, +} + + def saved_alert(content: bytes) -> Alert: view: Final = AlertView.model_validate_json(content) return Alert.model_validate(view.model_dump(exclude={"resource_name", "history"})) @@ -48,6 +58,17 @@ class Alerts: def detection(self, kind: AlertType = "spend", day: int = 0) -> AlertDetection: start: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + timedelta(days=day) + if kind == "prompt_cache": + return AlertDetection.model_validate( + { + "type": kind, + "resource_type": "deployment", + "resource_id": self.resource_id, + "window_start": start, + "window_end": start + timedelta(seconds=3), + "evidence": {**PROMPT_CACHE_EVIDENCE, "prefix_fingerprint": f"prefix-{day}"}, + } + ) evidence: Final = ( {"deprecation_date": "2026-03-01", "replacement_model": "example/replacement"} if kind == "model_deprecation" @@ -88,7 +109,7 @@ async def alerts() -> AsyncIterator[Alerts]: await client.disconnect() -@pytest.mark.parametrize("kind", ("spend", "traffic", "latency", "reliability", "model_deprecation")) +@pytest.mark.parametrize("kind", ("spend", "traffic", "latency", "reliability", "model_deprecation", "prompt_cache")) @pytest.mark.asyncio async def test_each_alert_type_round_trips_through_api_and_status_changes( alerts: Alerts, gateway: Gateway, kind: AlertType @@ -329,11 +350,36 @@ async def test_type_filter_applies_before_cursor_limit(alerts: Alerts, gateway: following: Final = gateway.request("GET", "/alerts", params={"type": "traffic", "limit": "1", "cursor": newest.id}) assert following.status_code == 200, following.text assert AlertViewPage.model_validate_json(following.content).data[0].id == oldest.id + cache: Final = await alerts.repository.record(alerts.detection("prompt_cache")) performance: Final = gateway.request("GET", "/alerts", params={"type": "performance"}) assert performance.status_code == 200, performance.text - assert all( - alert.type != "model_deprecation" for alert in AlertViewPage.model_validate_json(performance.content).data - ) + assert {alert.type for alert in AlertViewPage.model_validate_json(performance.content).data} <= { + "spend", + "traffic", + "latency", + "reliability", + } + incidents: Final = gateway.request("GET", "/alerts", params={"type": "prompt_cache"}) + assert incidents.status_code == 200, incidents.text + assert [alert.id for alert in AlertViewPage.model_validate_json(incidents.content).data] == [cache.id] + + +@pytest.mark.asyncio +async def test_a_dismissed_prompt_cache_incident_stays_the_only_one_for_its_configuration(alerts: Alerts) -> None: + first: Final = await alerts.repository.record(alerts.detection("prompt_cache")) + dismissed: Final = await alerts.repository.set_status(int(first.id), "dismissed", "reviewer") + assert await alerts.repository.record(alerts.detection("prompt_cache", day=3)) == dismissed + assert read_rows('SELECT count(*) AS count FROM "LiteLLM_Alert" WHERE resource_id=%s', (alerts.resource_id,)) == [ + {"count": 1} + ] + + +@pytest.mark.asyncio +async def test_daily_summaries_carry_metric_alerts_and_never_prompt_cache_alerts(alerts: Alerts) -> None: + spend: Final = await alerts.repository.record(alerts.detection("spend")) + cache: Final = await alerts.repository.record(alerts.detection("prompt_cache")) + assert [summary.id for summary in await alerts.repository.summaries(int(spend.id) - 1, int(spend.id))] == [spend.id] + assert await alerts.repository.summaries(int(cache.id) - 1, int(cache.id)) == () @pytest.mark.asyncio diff --git a/tests/integration/database/test_performance_alerts.py b/tests/integration/database/test_performance_alerts.py index 7b529a2a754..d1602554076 100644 --- a/tests/integration/database/test_performance_alerts.py +++ b/tests/integration/database/test_performance_alerts.py @@ -13,7 +13,7 @@ import pytest_asyncio from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes from psycopg import sql -from litellm.models.alert import AlertDetection, DeprecationEvidence +from litellm.models.alert import AlertDetection, DeprecationEvidence, PromptCacheEvidence from litellm.models.performance_alert import FREE_ALERT_LIMIT, SweepOwnershipLostError, SweepState from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.spend_tracking.performance_alerts import run_performance_alerts @@ -331,7 +331,9 @@ async def test_admin_can_opt_out_while_a_detection_batch_holds_the_state_lock( @pytest.mark.asyncio -async def test_existing_performance_alerts_consume_allowance_but_deprecations_do_not(fixture: Fixture) -> None: +async def test_existing_performance_alerts_consume_allowance_but_deprecations_and_prompt_cache_do_not( + fixture: Fixture, +) -> None: start: Final = datetime(2026, 1, 8, tzinfo=timezone.utc) await fixture.alerts.record( AlertDetection( @@ -343,6 +345,24 @@ async def test_existing_performance_alerts_consume_allowance_but_deprecations_do evidence=DeprecationEvidence(deprecation_date=DAY + timedelta(days=90)), ) ) + await fixture.alerts.record( + AlertDetection( + type="prompt_cache", + resource_type="deployment", + resource_id=f"{fixture.prefix}-deployment", + window_start=start, + window_end=start + timedelta(seconds=3), + evidence=PromptCacheEvidence( + config_fingerprint="config-1", + prefix_fingerprint="prefix-1", + provider="anthropic", + provider_model="claude-sonnet-5-5", + consecutive_misses=3, + proven_write_tokens=2048, + ttl_seconds=300, + ), + ) + ) assert (await fixture.repository.monitoring()).alerts_used == 0 _seed("team", f"{fixture.prefix}-a", Day(spend=10), Day(spend=100)) await fixture.sweep() diff --git a/tests/unit/models/test_alert.py b/tests/unit/models/test_alert.py index cf915c530b8..97c10af692d 100644 --- a/tests/unit/models/test_alert.py +++ b/tests/unit/models/test_alert.py @@ -24,6 +24,7 @@ DETECTION: Final = { {"resource_id": ""}, {"resource_type": "key", "resource_id": "sk-example-not-a-real-key"}, {"resource_type": "gateway", "resource_id": "another-gateway"}, + {"resource_type": "deployment", "resource_id": "claude-east"}, { "resource_type": "gateway", "resource_id": "gateway", @@ -66,3 +67,58 @@ def test_identity_uses_instants_not_timezone_or_evidence() -> None: } ) assert original.identity() == revised.identity() + + +CACHE_EVIDENCE: Final[Mapping[str, object]] = { + "config_fingerprint": "config-1", + "prefix_fingerprint": "prefix-1", + "provider": "anthropic", + "provider_model": "claude-test", + "consecutive_misses": 3, + "proven_write_tokens": 2048, + "ttl_seconds": 300, +} +PROMPT_CACHE: Final[Mapping[str, object]] = { + "type": "prompt_cache", + "resource_type": "deployment", + "resource_id": "claude-east", + "window_start": "2026-01-08T00:00:00Z", + "window_end": "2026-01-08T00:00:03Z", + "evidence": CACHE_EVIDENCE, +} + + +@pytest.mark.parametrize( + "change", + ( + {"resource_type": "model"}, + {"evidence": DETECTION["evidence"]}, + {"type": "spend"}, + {"evidence": {**CACHE_EVIDENCE, "consecutive_misses": 0}}, + {"evidence": {**CACHE_EVIDENCE, "provider": "bedrock"}}, + {"evidence": {**CACHE_EVIDENCE, "ttl_seconds": 3600}}, + {"evidence": {**CACHE_EVIDENCE, "prompt": "never retained"}}, + ), +) +def test_prompt_cache_detection_accepts_only_its_deployment_resource_and_supported_evidence( + change: Mapping[str, object], +) -> None: + with pytest.raises(ValidationError): + AlertDetection.model_validate({**PROMPT_CACHE, **change}) + + +def test_prompt_cache_identity_is_one_per_deployment_configuration_whatever_window_found_it() -> None: + first: Final = AlertDetection.model_validate(PROMPT_CACHE) + later: Final = AlertDetection.model_validate( + { + **PROMPT_CACHE, + "window_start": "2026-02-08T00:00:00Z", + "window_end": "2026-02-08T00:00:09Z", + "evidence": {**CACHE_EVIDENCE, "prefix_fingerprint": "prefix-2", "consecutive_misses": 4}, + } + ) + upgraded: Final = AlertDetection.model_validate( + {**PROMPT_CACHE, "evidence": {**CACHE_EVIDENCE, "config_fingerprint": "config-2"}} + ) + assert first.identity() == later.identity() + assert first.identity() != upgraded.identity() diff --git a/tests/unit/repositories/test_alert_repository.py b/tests/unit/repositories/test_alert_repository.py new file mode 100644 index 00000000000..c01fbb44638 --- /dev/null +++ b/tests/unit/repositories/test_alert_repository.py @@ -0,0 +1,104 @@ +from collections.abc import AsyncIterator, Mapping, Sequence +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +from litellm.models.alert import Alert, AlertDetection +from litellm.repositories.alert_repository import AlertRepository + +_START: Final = datetime(2026, 1, 8, tzinfo=timezone.utc) +_DETECTION: Final = AlertDetection.model_validate( + { + "type": "prompt_cache", + "resource_type": "deployment", + "resource_id": "claude-east", + "window_start": _START, + "window_end": _START + timedelta(seconds=3), + "evidence": { + "config_fingerprint": "config-1", + "prefix_fingerprint": "prefix-1", + "provider": "anthropic", + "provider_model": "claude-test", + "consecutive_misses": 3, + "proven_write_tokens": 2048, + "ttl_seconds": 300, + }, + } +) + + +class _Row: + def __init__(self, values: Mapping[str, object]) -> None: + self._values: Final = dict(values) + + def dict(self) -> dict[str, object]: + return self._values + + +def _row(status: str) -> _Row: + return _Row( + { + **_DETECTION.model_dump(), + "id": 7, + "detection_key": "key", + "status": status, + "created_at": _START, + "updated_at": _START, + "updated_by": None, + } + ) + + +class _Transaction: + def __init__(self) -> None: + self.queries: tuple[str, ...] = () + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + self.queries = (*self.queries, query) + return () + + +def _repository(transaction: _Transaction) -> tuple[AlertRepository, MagicMock]: + client: Final = MagicMock() + + @asynccontextmanager + async def tx(*, timeout: timedelta) -> AsyncIterator[_Transaction]: + yield transaction + + client.writer_db.tx = tx + return AlertRepository(client), client.writer_db.litellm_alert + + +async def test_a_later_detection_of_a_dismissed_incident_returns_it_unchanged() -> None: + """A dismissed incident stays the only one for its deployment and configuration, even when a later window + and another prefix find the same problem.""" + from prisma.errors import UniqueViolationError + + later: Final = _DETECTION.model_copy( + update={ + "window_start": _START + timedelta(hours=2), + "window_end": _START + timedelta(hours=2, seconds=3), + "evidence": _DETECTION.evidence.model_copy(update={"prefix_fingerprint": "prefix-2"}), + } + ) + repository, table = _repository(_Transaction()) + table.create = AsyncMock(side_effect=UniqueViolationError({}, message="detection_key")) + table.find_unique = AsyncMock(return_value=_row("dismissed")) + await repository.record(_DETECTION) + saved: Final = await repository.record(later) + first_key, later_key = (call.kwargs["data"]["detection_key"] for call in table.create.await_args_list) + assert first_key == later_key + assert saved.status == "dismissed" + table.update.assert_not_called() + + +async def test_prompt_cache_alerts_are_returned_without_looking_up_metric_history() -> None: + transaction: Final = _Transaction() + repository, _ = _repository(transaction) + incident: Final = Alert.model_validate( + {**_DETECTION.model_dump(), "id": "7", "status": "open", "created_at": _START, "updated_at": _START} + ) + (view,) = await repository.enrich((incident,)) + assert transaction.queries == () + assert (view.id, view.history) == ("7", None) diff --git a/tests/unit/repositories/test_performance_alert_repository.py b/tests/unit/repositories/test_performance_alert_repository.py index a7c85c770eb..57bf2630e3f 100644 --- a/tests/unit/repositories/test_performance_alert_repository.py +++ b/tests/unit/repositories/test_performance_alert_repository.py @@ -185,7 +185,7 @@ async def test_first_read_initializes_the_allowance_once(enterprise: bool) -> No assert first.active assert await repository.monitoring() == first assert execute.await_count == 2 - execute.assert_awaited_with(ANY, 5) + execute.assert_awaited_with(ANY, 5, ANY) @pytest.mark.asyncio