diff --git a/litellm/models/alert.py b/litellm/models/alert.py index 4db127bf121..ae7463c92fc 100644 --- a/litellm/models/alert.py +++ b/litellm/models/alert.py @@ -5,10 +5,12 @@ from typing import Annotated, Final, Literal, Protocol, TypeAlias from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, field_validator, model_validator from typing_extensions import Self -AlertType: TypeAlias = Literal["spend", "traffic", "latency", "reliability", "model_deprecation"] +MetricAlertType: TypeAlias = Literal["spend", "traffic", "latency", "reliability"] +AlertType: TypeAlias = MetricAlertType | Literal["model_deprecation", "prompt_cache"] AlertFilter: TypeAlias = AlertType | Literal["performance"] AlertStatus: TypeAlias = Literal["open", "resolved", "dismissed"] -AlertResourceType: TypeAlias = Literal["gateway", "key", "team", "user", "model"] +AlertResourceType: TypeAlias = Literal["gateway", "key", "team", "user", "model", "deployment"] +METRIC_ALERT_TYPES: Final[tuple[MetricAlertType, ...]] = ("spend", "traffic", "latency", "reliability") MAX_ALERT_ID: Final = 2**63 - 1 @@ -41,6 +43,22 @@ class DeprecationEvidence(BaseModel): ) +class PromptCacheEvidence(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + config_fingerprint: str = Field(min_length=1, max_length=64, description="Cache-relevant deployment settings.") + prefix_fingerprint: str = Field(min_length=1, max_length=64, description="Keyed hash of the cached prefix.") + provider: Literal["anthropic"] = Field(description="Provider that served the requests.") + provider_model: str = Field(min_length=1, max_length=1024, description="Model the provider served.") + consecutive_misses: int = Field( + ge=1, + le=MAX_ALERT_ID, + description="Observed requests that reported no cache read or write, not necessarily consecutive.", + ) + proven_write_tokens: int = Field(ge=1, le=MAX_ALERT_ID, description="Tokens the proving cache write stored.") + ttl_seconds: Literal[300] = Field(description="Cache lifetime of the prefix's breakpoint.") + + class AlertSummary(BaseModel): model_config = ConfigDict(frozen=True) @@ -75,7 +93,9 @@ class AlertDetection(BaseModel): window_end: AwareDatetime = Field( description="Exclusive end of the observed window, normalized to UTC milliseconds." ) - evidence: MetricEvidence | DeprecationEvidence = Field(description="Small saved snapshot explaining the alert.") + evidence: MetricEvidence | DeprecationEvidence | PromptCacheEvidence = Field( + description="Small saved snapshot explaining the alert." + ) @field_validator("window_start", "window_end") @classmethod @@ -96,6 +116,12 @@ class AlertDetection(BaseModel): if not isinstance(self.evidence, DeprecationEvidence) or self.resource_type != "model": raise ValueError("Model deprecations require a model resource and deprecation evidence") return self + if self.type == "prompt_cache": + if not isinstance(self.evidence, PromptCacheEvidence) or self.resource_type != "deployment": + raise ValueError("Prompt cache alerts require a deployment resource and prompt cache evidence") + return self + if self.resource_type == "deployment": + raise ValueError("Only prompt cache alerts use a deployment resource") expected_unit: Final = { "spend": "usd", "traffic": "requests", @@ -109,6 +135,10 @@ class AlertDetection(BaseModel): return self def identity(self) -> tuple[str, ...]: + """One prompt cache incident per deployment and configuration, whatever window or prefix found it, so a + dismissed or resolved one is never reopened by a later detection.""" + if isinstance(self.evidence, PromptCacheEvidence): + return (self.type, self.resource_type, self.resource_id, self.evidence.config_fingerprint) return ( self.type, self.resource_type, diff --git a/litellm/repositories/alert_repository.py b/litellm/repositories/alert_repository.py index 6b878fa6afc..cddafa761ad 100644 --- a/litellm/repositories/alert_repository.py +++ b/litellm/repositories/alert_repository.py @@ -16,6 +16,7 @@ from typing_extensions import LiteralString from litellm._logging import verbose_proxy_logger from litellm.models.alert import ( + METRIC_ALERT_TYPES, Alert, AlertDetection, AlertFilter, @@ -203,10 +204,11 @@ class AlertRepository: LEFT JOIN "LiteLLM_VerificationToken" k ON a.resource_type = 'key' AND k.token = a.resource_id LEFT JOIN "LiteLLM_TeamTable" t ON a.resource_type = 'team' AND t.team_id = a.resource_id LEFT JOIN "LiteLLM_UserTable" u ON a.resource_type = 'user' AND u.user_id = a.resource_id - WHERE a.id > $1 AND a.id <= $2 AND a.status = 'open' AND a.type != 'model_deprecation' + WHERE a.id > $1 AND a.id <= $2 AND a.status = 'open' AND a.type = ANY($3::text[]) ORDER BY a.id DESC LIMIT 5""", after, through, + list(METRIC_ALERT_TYPES), ) return _SUMMARIES.validate_python(rows) @@ -234,7 +236,8 @@ class AlertRepository: self, status: AlertStatus, limit: int, cursor: int | None, alert_type: AlertFilter | None = None ) -> AlertPage: size: Final = max(1, min(limit, 100)) - category: Final = {"not": "model_deprecation"} if alert_type == "performance" else alert_type + # bounded-ok: METRIC_ALERT_TYPES is the fixed four metric categories + category: Final = {"in": list(METRIC_ALERT_TYPES)} if alert_type == "performance" else alert_type rows: Final = await self._table.find_many( where={ "status": status, @@ -269,11 +272,13 @@ class AlertRepository: "start": (alert.window_start.date() - timedelta(days=7)).isoformat(), } for alert in alerts - if alert.type != "model_deprecation" + if alert.type in METRIC_ALERT_TYPES ) if not requested: return tuple(AlertView.model_validate(alert.model_dump()) for alert in alerts) - scopes: Final = frozenset[AlertResourceType](alert.resource_type for alert in alerts) + scopes: Final = frozenset[AlertResourceType]( + alert.resource_type for alert in alerts if alert.type in METRIC_ALERT_TYPES + ) histories: Final = " UNION ALL ".join(_history_sql(scope) for scope in scopes) query: Final = f"""WITH requested AS ( SELECT * FROM jsonb_to_recordset($1::jsonb) diff --git a/litellm/repositories/performance_alert_repository.py b/litellm/repositories/performance_alert_repository.py index 9eaaed77d3c..2f60c18d172 100644 --- a/litellm/repositories/performance_alert_repository.py +++ b/litellm/repositories/performance_alert_repository.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, Protocol, cast # noqa: TID251 # Prism from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError from typing_extensions import LiteralString +from litellm.models.alert import METRIC_ALERT_TYPES from litellm.models.performance_alert import ( BASELINE_DAYS, FREE_ALERT_LIMIT, @@ -178,9 +179,10 @@ class PerformanceAlertRepository: await transaction.execute_raw( """UPDATE "LiteLLM_PerformanceAlertState" SET alerts_used = ( SELECT COUNT(*) FROM (SELECT 1 FROM "LiteLLM_Alert" - WHERE type != 'model_deprecation' LIMIT $1) existing + WHERE type = ANY($2::text[]) LIMIT $1) existing ) WHERE id = 1 AND alerts_used IS NULL""", FREE_ALERT_LIMIT, + list(METRIC_ALERT_TYPES), ) return await self._monitoring(transaction, lock=lock) return AlertMonitoring( diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx index 733641fdb17..679177de310 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/AlertDetail.tsx @@ -1,10 +1,11 @@ -import { ArrowLeft, KeyRound, Network, Server, User, Users } from "lucide-react"; +import { ArrowLeft, Cpu, KeyRound, Network, Server, User, Users } from "lucide-react"; import { useTranslation } from "react-i18next"; import { buttonVariants } from "@/components/ui/button"; import RecommendedReplacement from "@/components/ModelDeprecationAlerts/RecommendedReplacement"; import { migrationHref } from "@/components/ModelDeprecationAlerts/migrationRoute"; import AlertActions from "./AlertActions"; import AlertChart from "./AlertChart"; +import PromptCacheEvidence from "./PromptCacheEvidence"; import { AlertItem, dateLabel, presentAlert } from "./alertPresentation"; export default function AlertDetail({ @@ -18,7 +19,14 @@ export default function AlertDetail({ }) { const { t } = useTranslation(); const alert = presentAlert(item); - const Icon = { key: KeyRound, team: Users, user: User, model: Network, gateway: Server }[alert.resourceType]; + const Icon = { + key: KeyRound, + team: Users, + user: User, + model: Network, + gateway: Server, + deployment: Cpu, + }[alert.resourceType]; return (
@@ -158,6 +166,9 @@ function AlertsInbox({ auth }: { auth: ReturnType }) { {showDeprecations && deprecations.isError && ( void deprecations.refetch()}>{t("alerts.unified.deprecationsError")} )} + {showPromptCache && promptCache.isError && ( + void promptCache.refetch()}>{t("alerts.unified.promptCacheError")} + )} {showPerformance && list.isError && ( void list.refetch()}>{t("alerts.unified.performanceError")} )} @@ -194,14 +205,16 @@ function AlertsInbox({ auth }: { auth: ReturnType }) { ); })} - {showPerformance && list.hasNextPage && ( + {pagedLists.some((query) => query.hasNextPage) && ( )} diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/PromptCacheEvidence.tsx b/litellm/ui/litellm-dashboard/src/components/Alerts/PromptCacheEvidence.tsx new file mode 100644 index 00000000000..1c0091f9db8 --- /dev/null +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/PromptCacheEvidence.tsx @@ -0,0 +1,44 @@ +import { useTranslation } from "react-i18next"; +import type { PromptCacheDetails } from "./alertPresentation"; + +const timeFormat = { dateStyle: "medium", timeStyle: "medium", timeZone: "UTC" } as const; + +export default function PromptCacheEvidence({ details }: { details: PromptCacheDetails }) { + const { t } = useTranslation(); + const facts: [string, string][] = [ + [t("alerts.unified.promptCache.deployment"), details.deploymentId], + [t("alerts.unified.promptCache.model"), details.model], + [t("alerts.unified.promptCache.misses"), details.misses.toLocaleString()], + [ + t("alerts.unified.promptCache.provenWrite"), + t("alerts.unified.promptCache.provenWriteTokens", { count: details.provenWriteTokens }), + ], + [ + t("alerts.unified.promptCache.ttl"), + t("alerts.unified.promptCache.ttlMinutes", { count: details.ttlSeconds / 60 }), + ], + [ + t("alerts.unified.promptCache.window"), + t("alerts.unified.promptCache.windowValue", { + start: new Date(details.firstMiss).toLocaleString(undefined, timeFormat), + end: new Date(details.lastMiss).toLocaleString(undefined, timeFormat), + }), + ], + ]; + return ( +
+

+ {t("alerts.unified.promptCache.evidenceHeading")} +

+
+ {facts.map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+

{t("alerts.unified.promptCache.scope")}

+
+ ); +} diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts b/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts index aa92ca8fc20..47a473260a4 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.test.ts @@ -31,6 +31,48 @@ describe("alert presentation", () => { expect(presentation.description).toContain(formatted); expect(presentation.baseline).toBe(baseline); }); + it("presents a prompt cache incident without reading it as a metric or deprecation alert", () => { + const presentation = presentAlert({ + source: "performance", + alert: { + ...saved, + type: "prompt_cache", + resource_type: "deployment", + resource_id: "claude-east", + evidence: { + config_fingerprint: "config-1", + prefix_fingerprint: "prefix-1", + provider: "anthropic", + provider_model: "claude-test", + consecutive_misses: 3, + proven_write_tokens: 2048, + ttl_seconds: 300, + }, + }, + }); + expect(presentation.title).toBe("Prompt cache usage advisory for claude-east"); + expect(presentation.description).toContain("3 observed requests to claude-test reported no cache usage"); + expect(presentation.description).toContain("not necessarily consecutive and do not prove a caching outage"); + expect(presentation.description).not.toContain("prefix-1"); + expect([presentation.resourceType, presentation.href, presentation.history]).toEqual([ + "deployment", + undefined, + null, + ]); + const expectedDetails = { + deploymentId: "claude-east", + model: "claude-test", + misses: 3, + provenWriteTokens: 2048, + ttlSeconds: 300, + firstMiss: saved.window_start, + lastMiss: saved.window_end, + }; + expect(presentation.promptCache).toEqual(expectedDetails); + }); + it("keeps metric alerts on their chart", () => { + expect(presentAlert({ source: "performance", alert: saved }).promptCache).toBeNull(); + }); it("links to the resource with an inclusive eight-day window", () => { expect(activityLink(saved)).toBe( "/ui/teams?team=example-team&detail_tab=activity&activity_from=2026-01-01&activity_to=2026-01-08", diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts b/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts index 79a5d84a2a4..ef408722236 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/alertPresentation.ts @@ -1,4 +1,4 @@ -import type { components } from "@/lib/http/schema"; +import type { components, operations } from "@/lib/http/schema"; import i18n from "@/i18n/i18n"; import type { DeprecationAlert } from "@/components/ModelDeprecationAlerts/deprecationAlerts"; import { effectiveAlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; @@ -11,6 +11,15 @@ export type AlertItem = | { source: "deprecation"; alert: DeprecationAlert }; export type MetricUnit = "usd" | "requests" | "milliseconds" | "ratio"; export type HistoryPoint = { date: string; value: number | null }; +export type PromptCacheDetails = { + deploymentId: string; + model: string; + misses: number; + provenWriteTokens: number; + ttlSeconds: number; + firstMiss: string; + lastMiss: string; +}; export const ALERT_TYPES = [ "all", "performance", @@ -21,6 +30,23 @@ export const ALERT_TYPES = [ "reliability", ] as const; export type AlertFilter = (typeof ALERT_TYPES)[number]; +export type ListedAlertType = NonNullable< + NonNullable["type"] +>; +export function listedAlertType(filter: AlertFilter): ListedAlertType | null { + if (filter === "model-deprecations") return null; + return filter === "all" ? "performance" : filter; +} +const METRIC_LABEL_KEYS = { + usd: "alerts.unified.metrics.spend", + requests: "alerts.unified.metrics.traffic", + milliseconds: "alerts.unified.metrics.latency", + ratio: "alerts.unified.metrics.reliability", +} as const; +function linkLabel(resource: SavedAlert["resource_type"]): string { + if (resource === "deployment") return ""; + return i18n.t(`alerts.unified.links.${resource}`); +} export const TYPE_LABEL_KEYS = { all: "alerts.unified.types.all", performance: "alerts.unified.types.performance", @@ -98,6 +124,7 @@ export function presentAlert(item: AlertItem) { [i18n.t("alerts.unified.lastUsed"), alert.lastUsedDate], ], chartTitle: i18n.t("alerts.usageChart.title"), + promptCache: null, }; } const alert = item.alert; @@ -106,6 +133,38 @@ export function presentAlert(item: AlertItem) { (alert.resource_type === "key" ? i18n.t("alerts.unified.keyFallback", { id: alert.resource_id.slice(0, 8) }) : alert.resource_id); + if ("consecutive_misses" in alert.evidence) { + const evidence = alert.evidence; + const promptCache: PromptCacheDetails = { + deploymentId: alert.resource_id, + model: evidence.provider_model, + misses: evidence.consecutive_misses, + provenWriteTokens: evidence.proven_write_tokens, + ttlSeconds: evidence.ttl_seconds, + firstMiss: alert.window_start, + lastMiss: alert.window_end, + }; + return { + id: alert.id, + title: i18n.t("alerts.unified.promptCache.title", { name }), + description: i18n.t("alerts.unified.promptCache.description", { + misses: evidence.consecutive_misses.toLocaleString(), + model: evidence.provider_model, + }), + status: alert.status, + resourceType: alert.resource_type, + resourceName: name, + href: undefined, + linkLabel: "", + unit: "requests" as MetricUnit, + history: null, + baseline: undefined, + observed: alert.window_start.slice(0, 10), + dates: [[i18n.t("alerts.unified.firstDetected"), alert.created_at]], + chartTitle: i18n.t("alerts.usageChart.title"), + promptCache, + }; + } if (!("value" in alert.evidence)) { return { id: alert.id, @@ -122,11 +181,12 @@ export function presentAlert(item: AlertItem) { observed: alert.window_start.slice(0, 10), dates: [[i18n.t("alerts.unified.retires"), alert.evidence.deprecation_date]], chartTitle: i18n.t("alerts.usageChart.title"), + promptCache: null, }; } const evidence = alert.evidence; const observed = alert.window_start.slice(0, 10); - const label = i18n.t(`alerts.unified.metrics.${alert.type}`); + const label = i18n.t(METRIC_LABEL_KEYS[evidence.unit]); const direction = evidence.value < evidence.baseline ? "fell" : "rose"; return { id: alert.id, @@ -151,7 +211,7 @@ export function presentAlert(item: AlertItem) { resourceType: alert.resource_type, resourceName: name, href: activityLink(alert), - linkLabel: i18n.t(`alerts.unified.links.${alert.resource_type}`), + linkLabel: linkLabel(alert.resource_type), unit: evidence.unit, history: alert.history, baseline: evidence.baseline, @@ -161,6 +221,7 @@ export function presentAlert(item: AlertItem) { [i18n.t("alerts.unified.firstDetected"), alert.created_at], ], chartTitle: i18n.t(`alerts.unified.chart.titles.${evidence.unit}`), + promptCache: null, }; } export type AlertPresentation = ReturnType; diff --git a/litellm/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts b/litellm/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts index 70813ef2e93..ea79bd26460 100644 --- a/litellm/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts +++ b/litellm/ui/litellm-dashboard/src/components/Alerts/useAlerts.ts @@ -3,17 +3,17 @@ import { useInfiniteQuery, useQuery } from "@tanstack/react-query"; import { backgroundFetchClient } from "@/lib/http/api"; import { getProxyBaseUrl } from "@/components/networking"; import { AlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; -import type { AlertFilter } from "./alertPresentation"; +import { listedAlertType, type AlertFilter } from "./alertPresentation"; -export function useAlerts(scope: string, enabled: boolean, status: AlertStatus, filter: AlertFilter) { +export function useAlerts(scope: string, enabled: boolean, status: AlertStatus, filter: AlertFilter | "prompt_cache") { + const type = filter === "prompt_cache" ? filter : listedAlertType(filter); const options = { queryKey: ["alerts", getProxyBaseUrl(), scope, status, filter], - enabled: enabled && filter !== "model-deprecations", + enabled: enabled && type !== null, initialPageParam: undefined as string | undefined, queryFn: async ({ pageParam, signal }: { pageParam: string | undefined; signal: AbortSignal }) => { - const type = filter === "all" || filter === "model-deprecations" ? "performance" : filter; const { data } = await backgroundFetchClient.GET("/alerts", { - params: { query: { status, type, limit: 25, cursor: pageParam } }, + params: { query: { status, type: type ?? undefined, limit: 25, cursor: pageParam } }, signal, }); if (!data) throw new Error("Could not load alerts"); diff --git a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json index e25275d0612..8f755b63373 100644 --- a/litellm/ui/litellm-dashboard/src/i18n/locales/en.json +++ b/litellm/ui/litellm-dashboard/src/i18n/locales/en.json @@ -5868,6 +5868,23 @@ }, "retiredTitle": "{{name}} retired on {{date}}", "retiresTitle": "{{name}} retires on {{date}}", + "promptCache": { + "title": "Prompt cache usage advisory for {{name}}", + "description": "{{misses}} observed requests to {{model}} reported no cache usage for a prefix the provider had cached before. Delayed, missing or unlogged observations mean these requests are not necessarily consecutive and do not prove a caching outage.", + "evidenceHeading": "What was observed", + "deployment": "Deployment", + "model": "Model", + "misses": "Observed requests with no cache usage", + "provenWrite": "Earlier cache write that proved the prefix", + "provenWriteTokens_one": "{{count}} token", + "provenWriteTokens_other": "{{count}} tokens", + "ttl": "Cache lifetime", + "ttlMinutes_one": "{{count}} minute", + "ttlMinutes_other": "{{count}} minutes", + "window": "Misses observed", + "windowValue": "{{start}} to {{end}} UTC", + "scope": "Only non-streaming Anthropic Chat Completions requests with one five-minute cache breakpoint are checked, after the provider confirmed a cache write for that prefix. Prefixes never confirmed as cached are not assessed." + }, "deprecationDescription": "{{requests}} recorded requests. Last used {{date}} UTC.", "retired": "Retired", "retires": "Retires", @@ -5913,7 +5930,8 @@ "milliseconds": "Mean response time per day", "ratio": "Daily error rate" } - } + }, + "promptCacheError": "Could not load prompt cache alerts." }, "page": { "tabs": { diff --git a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts index 681a91b3809..ada43316510 100644 --- a/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/litellm/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27698,7 +27698,7 @@ export interface components { * Evidence * @description Small saved snapshot explaining the alert. */ - evidence: components["schemas"]["MetricEvidence"] | components["schemas"]["DeprecationEvidence"]; + evidence: components["schemas"]["MetricEvidence"] | components["schemas"]["DeprecationEvidence"] | components["schemas"]["PromptCacheEvidence"]; /** * Id * @description Opaque alert ID, also used as a pagination cursor. @@ -27714,7 +27714,7 @@ export interface components { * @description Kind of affected resource. * @enum {string} */ - resource_type: "gateway" | "key" | "team" | "user" | "model"; + resource_type: "gateway" | "key" | "team" | "user" | "model" | "deployment"; /** * Status * @description Current review status. @@ -27724,9 +27724,8 @@ export interface components { /** * Type * @description Built-in alert category. - * @enum {string} */ - type: "spend" | "traffic" | "latency" | "reliability" | "model_deprecation"; + type: ("spend" | "traffic" | "latency" | "reliability") | ("model_deprecation" | "prompt_cache"); /** * Updated At * Format: date-time @@ -27829,7 +27828,7 @@ export interface components { * Evidence * @description Small saved snapshot explaining the alert. */ - evidence: components["schemas"]["MetricEvidence"] | components["schemas"]["DeprecationEvidence"]; + evidence: components["schemas"]["MetricEvidence"] | components["schemas"]["DeprecationEvidence"] | components["schemas"]["PromptCacheEvidence"]; /** * History * @description Current Usage values for the seven baseline days and observed day. Null if unavailable. @@ -27855,7 +27854,7 @@ export interface components { * @description Kind of affected resource. * @enum {string} */ - resource_type: "gateway" | "key" | "team" | "user" | "model"; + resource_type: "gateway" | "key" | "team" | "user" | "model" | "deployment"; /** * Status * @description Current review status. @@ -27865,9 +27864,8 @@ export interface components { /** * Type * @description Built-in alert category. - * @enum {string} */ - type: "spend" | "traffic" | "latency" | "reliability" | "model_deprecation"; + type: ("spend" | "traffic" | "latency" | "reliability") | ("model_deprecation" | "prompt_cache"); /** * Updated At * Format: date-time @@ -44680,6 +44678,46 @@ export interface components { prompt_id: string; prompt_info?: components["schemas"]["PromptInfo"] | null; }; + /** PromptCacheEvidence */ + PromptCacheEvidence: { + /** + * Config Fingerprint + * @description Cache-relevant deployment settings. + */ + config_fingerprint: string; + /** + * Consecutive Misses + * @description Observed requests that reported no cache read or write, not necessarily consecutive. + */ + consecutive_misses: number; + /** + * Prefix Fingerprint + * @description Keyed hash of the cached prefix. + */ + prefix_fingerprint: string; + /** + * Proven Write Tokens + * @description Tokens the proving cache write stored. + */ + proven_write_tokens: number; + /** + * Provider + * @description Provider that served the requests. + * @constant + */ + provider: "anthropic"; + /** + * Provider Model + * @description Model the provider served. + */ + provider_model: string; + /** + * Ttl Seconds + * @description Cache lifetime of the prefix's breakpoint. + * @constant + */ + ttl_seconds: 300; + }; /** PromptCacheOptions */ PromptCacheOptions: { /** Comparison Response Id */ @@ -56290,7 +56328,7 @@ export interface operations { /** @description next_cursor from the preceding page. */ cursor?: string | null; /** @description Alert category, or all performance alerts. */ - type?: ("spend" | "traffic" | "latency" | "reliability" | "model_deprecation") | "performance" | null; + type?: ("spend" | "traffic" | "latency" | "reliability") | ("model_deprecation" | "prompt_cache") | "performance" | null; }; header?: never; path?: never; diff --git a/tests/integration/database/test_alerts.py b/tests/integration/database/test_alerts.py index 06908c3a569..e15a24133a6 100644 --- a/tests/integration/database/test_alerts.py +++ b/tests/integration/database/test_alerts.py @@ -28,6 +28,16 @@ from tests.integration._support.client import Gateway from tests.integration._support.database import read_rows, write_rows +PROMPT_CACHE_EVIDENCE: Final = { + "config_fingerprint": "config-1", + "provider": "anthropic", + "provider_model": "claude-sonnet-5-5", + "consecutive_misses": 3, + "proven_write_tokens": 2048, + "ttl_seconds": 300, +} + + def saved_alert(content: bytes) -> Alert: view: Final = AlertView.model_validate_json(content) return Alert.model_validate(view.model_dump(exclude={"resource_name", "history"})) @@ -48,6 +58,17 @@ class Alerts: def detection(self, kind: AlertType = "spend", day: int = 0) -> AlertDetection: start: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + timedelta(days=day) + if kind == "prompt_cache": + return AlertDetection.model_validate( + { + "type": kind, + "resource_type": "deployment", + "resource_id": self.resource_id, + "window_start": start, + "window_end": start + timedelta(seconds=3), + "evidence": {**PROMPT_CACHE_EVIDENCE, "prefix_fingerprint": f"prefix-{day}"}, + } + ) evidence: Final = ( {"deprecation_date": "2026-03-01", "replacement_model": "example/replacement"} if kind == "model_deprecation" @@ -88,7 +109,7 @@ async def alerts() -> AsyncIterator[Alerts]: await client.disconnect() -@pytest.mark.parametrize("kind", ("spend", "traffic", "latency", "reliability", "model_deprecation")) +@pytest.mark.parametrize("kind", ("spend", "traffic", "latency", "reliability", "model_deprecation", "prompt_cache")) @pytest.mark.asyncio async def test_each_alert_type_round_trips_through_api_and_status_changes( alerts: Alerts, gateway: Gateway, kind: AlertType @@ -329,11 +350,36 @@ async def test_type_filter_applies_before_cursor_limit(alerts: Alerts, gateway: following: Final = gateway.request("GET", "/alerts", params={"type": "traffic", "limit": "1", "cursor": newest.id}) assert following.status_code == 200, following.text assert AlertViewPage.model_validate_json(following.content).data[0].id == oldest.id + cache: Final = await alerts.repository.record(alerts.detection("prompt_cache")) performance: Final = gateway.request("GET", "/alerts", params={"type": "performance"}) assert performance.status_code == 200, performance.text - assert all( - alert.type != "model_deprecation" for alert in AlertViewPage.model_validate_json(performance.content).data - ) + assert {alert.type for alert in AlertViewPage.model_validate_json(performance.content).data} <= { + "spend", + "traffic", + "latency", + "reliability", + } + incidents: Final = gateway.request("GET", "/alerts", params={"type": "prompt_cache"}) + assert incidents.status_code == 200, incidents.text + assert [alert.id for alert in AlertViewPage.model_validate_json(incidents.content).data] == [cache.id] + + +@pytest.mark.asyncio +async def test_a_dismissed_prompt_cache_incident_stays_the_only_one_for_its_configuration(alerts: Alerts) -> None: + first: Final = await alerts.repository.record(alerts.detection("prompt_cache")) + dismissed: Final = await alerts.repository.set_status(int(first.id), "dismissed", "reviewer") + assert await alerts.repository.record(alerts.detection("prompt_cache", day=3)) == dismissed + assert read_rows('SELECT count(*) AS count FROM "LiteLLM_Alert" WHERE resource_id=%s', (alerts.resource_id,)) == [ + {"count": 1} + ] + + +@pytest.mark.asyncio +async def test_daily_summaries_carry_metric_alerts_and_never_prompt_cache_alerts(alerts: Alerts) -> None: + spend: Final = await alerts.repository.record(alerts.detection("spend")) + cache: Final = await alerts.repository.record(alerts.detection("prompt_cache")) + assert [summary.id for summary in await alerts.repository.summaries(int(spend.id) - 1, int(spend.id))] == [spend.id] + assert await alerts.repository.summaries(int(cache.id) - 1, int(cache.id)) == () @pytest.mark.asyncio diff --git a/tests/integration/database/test_performance_alerts.py b/tests/integration/database/test_performance_alerts.py index 7b529a2a754..d1602554076 100644 --- a/tests/integration/database/test_performance_alerts.py +++ b/tests/integration/database/test_performance_alerts.py @@ -13,7 +13,7 @@ import pytest_asyncio from litellm_proxy_extras.request_log_indexes import ensure_request_log_indexes from psycopg import sql -from litellm.models.alert import AlertDetection, DeprecationEvidence +from litellm.models.alert import AlertDetection, DeprecationEvidence, PromptCacheEvidence from litellm.models.performance_alert import FREE_ALERT_LIMIT, SweepOwnershipLostError, SweepState from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.spend_tracking.performance_alerts import run_performance_alerts @@ -331,7 +331,9 @@ async def test_admin_can_opt_out_while_a_detection_batch_holds_the_state_lock( @pytest.mark.asyncio -async def test_existing_performance_alerts_consume_allowance_but_deprecations_do_not(fixture: Fixture) -> None: +async def test_existing_performance_alerts_consume_allowance_but_deprecations_and_prompt_cache_do_not( + fixture: Fixture, +) -> None: start: Final = datetime(2026, 1, 8, tzinfo=timezone.utc) await fixture.alerts.record( AlertDetection( @@ -343,6 +345,24 @@ async def test_existing_performance_alerts_consume_allowance_but_deprecations_do evidence=DeprecationEvidence(deprecation_date=DAY + timedelta(days=90)), ) ) + await fixture.alerts.record( + AlertDetection( + type="prompt_cache", + resource_type="deployment", + resource_id=f"{fixture.prefix}-deployment", + window_start=start, + window_end=start + timedelta(seconds=3), + evidence=PromptCacheEvidence( + config_fingerprint="config-1", + prefix_fingerprint="prefix-1", + provider="anthropic", + provider_model="claude-sonnet-5-5", + consecutive_misses=3, + proven_write_tokens=2048, + ttl_seconds=300, + ), + ) + ) assert (await fixture.repository.monitoring()).alerts_used == 0 _seed("team", f"{fixture.prefix}-a", Day(spend=10), Day(spend=100)) await fixture.sweep() diff --git a/tests/unit/models/test_alert.py b/tests/unit/models/test_alert.py index cf915c530b8..97c10af692d 100644 --- a/tests/unit/models/test_alert.py +++ b/tests/unit/models/test_alert.py @@ -24,6 +24,7 @@ DETECTION: Final = { {"resource_id": ""}, {"resource_type": "key", "resource_id": "sk-example-not-a-real-key"}, {"resource_type": "gateway", "resource_id": "another-gateway"}, + {"resource_type": "deployment", "resource_id": "claude-east"}, { "resource_type": "gateway", "resource_id": "gateway", @@ -66,3 +67,58 @@ def test_identity_uses_instants_not_timezone_or_evidence() -> None: } ) assert original.identity() == revised.identity() + + +CACHE_EVIDENCE: Final[Mapping[str, object]] = { + "config_fingerprint": "config-1", + "prefix_fingerprint": "prefix-1", + "provider": "anthropic", + "provider_model": "claude-test", + "consecutive_misses": 3, + "proven_write_tokens": 2048, + "ttl_seconds": 300, +} +PROMPT_CACHE: Final[Mapping[str, object]] = { + "type": "prompt_cache", + "resource_type": "deployment", + "resource_id": "claude-east", + "window_start": "2026-01-08T00:00:00Z", + "window_end": "2026-01-08T00:00:03Z", + "evidence": CACHE_EVIDENCE, +} + + +@pytest.mark.parametrize( + "change", + ( + {"resource_type": "model"}, + {"evidence": DETECTION["evidence"]}, + {"type": "spend"}, + {"evidence": {**CACHE_EVIDENCE, "consecutive_misses": 0}}, + {"evidence": {**CACHE_EVIDENCE, "provider": "bedrock"}}, + {"evidence": {**CACHE_EVIDENCE, "ttl_seconds": 3600}}, + {"evidence": {**CACHE_EVIDENCE, "prompt": "never retained"}}, + ), +) +def test_prompt_cache_detection_accepts_only_its_deployment_resource_and_supported_evidence( + change: Mapping[str, object], +) -> None: + with pytest.raises(ValidationError): + AlertDetection.model_validate({**PROMPT_CACHE, **change}) + + +def test_prompt_cache_identity_is_one_per_deployment_configuration_whatever_window_found_it() -> None: + first: Final = AlertDetection.model_validate(PROMPT_CACHE) + later: Final = AlertDetection.model_validate( + { + **PROMPT_CACHE, + "window_start": "2026-02-08T00:00:00Z", + "window_end": "2026-02-08T00:00:09Z", + "evidence": {**CACHE_EVIDENCE, "prefix_fingerprint": "prefix-2", "consecutive_misses": 4}, + } + ) + upgraded: Final = AlertDetection.model_validate( + {**PROMPT_CACHE, "evidence": {**CACHE_EVIDENCE, "config_fingerprint": "config-2"}} + ) + assert first.identity() == later.identity() + assert first.identity() != upgraded.identity() diff --git a/tests/unit/repositories/test_alert_repository.py b/tests/unit/repositories/test_alert_repository.py new file mode 100644 index 00000000000..c01fbb44638 --- /dev/null +++ b/tests/unit/repositories/test_alert_repository.py @@ -0,0 +1,104 @@ +from collections.abc import AsyncIterator, Mapping, Sequence +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +from litellm.models.alert import Alert, AlertDetection +from litellm.repositories.alert_repository import AlertRepository + +_START: Final = datetime(2026, 1, 8, tzinfo=timezone.utc) +_DETECTION: Final = AlertDetection.model_validate( + { + "type": "prompt_cache", + "resource_type": "deployment", + "resource_id": "claude-east", + "window_start": _START, + "window_end": _START + timedelta(seconds=3), + "evidence": { + "config_fingerprint": "config-1", + "prefix_fingerprint": "prefix-1", + "provider": "anthropic", + "provider_model": "claude-test", + "consecutive_misses": 3, + "proven_write_tokens": 2048, + "ttl_seconds": 300, + }, + } +) + + +class _Row: + def __init__(self, values: Mapping[str, object]) -> None: + self._values: Final = dict(values) + + def dict(self) -> dict[str, object]: + return self._values + + +def _row(status: str) -> _Row: + return _Row( + { + **_DETECTION.model_dump(), + "id": 7, + "detection_key": "key", + "status": status, + "created_at": _START, + "updated_at": _START, + "updated_by": None, + } + ) + + +class _Transaction: + def __init__(self) -> None: + self.queries: tuple[str, ...] = () + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + self.queries = (*self.queries, query) + return () + + +def _repository(transaction: _Transaction) -> tuple[AlertRepository, MagicMock]: + client: Final = MagicMock() + + @asynccontextmanager + async def tx(*, timeout: timedelta) -> AsyncIterator[_Transaction]: + yield transaction + + client.writer_db.tx = tx + return AlertRepository(client), client.writer_db.litellm_alert + + +async def test_a_later_detection_of_a_dismissed_incident_returns_it_unchanged() -> None: + """A dismissed incident stays the only one for its deployment and configuration, even when a later window + and another prefix find the same problem.""" + from prisma.errors import UniqueViolationError + + later: Final = _DETECTION.model_copy( + update={ + "window_start": _START + timedelta(hours=2), + "window_end": _START + timedelta(hours=2, seconds=3), + "evidence": _DETECTION.evidence.model_copy(update={"prefix_fingerprint": "prefix-2"}), + } + ) + repository, table = _repository(_Transaction()) + table.create = AsyncMock(side_effect=UniqueViolationError({}, message="detection_key")) + table.find_unique = AsyncMock(return_value=_row("dismissed")) + await repository.record(_DETECTION) + saved: Final = await repository.record(later) + first_key, later_key = (call.kwargs["data"]["detection_key"] for call in table.create.await_args_list) + assert first_key == later_key + assert saved.status == "dismissed" + table.update.assert_not_called() + + +async def test_prompt_cache_alerts_are_returned_without_looking_up_metric_history() -> None: + transaction: Final = _Transaction() + repository, _ = _repository(transaction) + incident: Final = Alert.model_validate( + {**_DETECTION.model_dump(), "id": "7", "status": "open", "created_at": _START, "updated_at": _START} + ) + (view,) = await repository.enrich((incident,)) + assert transaction.queries == () + assert (view.id, view.history) == ("7", None) diff --git a/tests/unit/repositories/test_performance_alert_repository.py b/tests/unit/repositories/test_performance_alert_repository.py index a7c85c770eb..57bf2630e3f 100644 --- a/tests/unit/repositories/test_performance_alert_repository.py +++ b/tests/unit/repositories/test_performance_alert_repository.py @@ -185,7 +185,7 @@ async def test_first_read_initializes_the_allowance_once(enterprise: bool) -> No assert first.active assert await repository.monitoring() == first assert execute.await_count == 2 - execute.assert_awaited_with(ANY, 5) + execute.assert_awaited_with(ANY, 5, ANY) @pytest.mark.asyncio