From d2b2e763745148ec2d134aca68a4402073714daa Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Thu, 5 Nov 2026 12:03:42 +0800 Subject: [PATCH] feat(proxy): add model deprecation alerts with daily requests by key (#50022) * feat(ui): show recent model deprecations in notification bell * feat(ui): simplify deprecation request trends and add model identity --- litellm/proxy/alerting_endpoints/__init__.py | 1 + .../model_deprecation_alerts.py | 109 ++++ litellm/proxy/proxy_server.py | 4 + litellm/types/proxy/model_deprecation.py | 38 +- .../test_model_deprecation_alerts.py | 180 +++++++ .../src/app/(dashboard)/alerts/page.tsx | 19 + .../hooks/models/invalidateModelQueries.ts | 9 - .../models/useModelDeprecations.test.tsx | 64 --- .../hooks/models/useModelDeprecations.ts | 33 -- .../app/(dashboard)/hooks/models/useModels.ts | 5 +- .../src/app/(dashboard)/legacyPageRoutes.ts | 1 + .../AllModelsTab.integration.test.tsx | 232 --------- .../components/AllModelsTab.test.tsx | 9 +- .../components/AllModelsTab.tsx | 11 +- .../ModelDeprecationBanner.test.tsx | 477 ------------------ .../components/ModelDeprecationBanner.tsx | 98 ---- ...riceDataManagementTab.integration.test.tsx | 83 --- .../PriceDataManagementTab.test.tsx | 8 +- .../components/PriceDataManagementTab.tsx | 6 +- .../components/deprecationBannerText.test.ts | 140 ----- .../components/deprecationBannerText.ts | 96 ---- .../page.integration.test.tsx | 55 +- .../models-and-endpoints/page.test.tsx | 9 - .../(dashboard)/models-and-endpoints/page.tsx | 7 +- .../panels/AddModelPanel.integration.test.tsx | 78 +-- .../panels/AddModelPanel.tsx | 3 +- .../chat_ui/AgentBuilderView.test.tsx | 97 +--- .../components/chat_ui/AgentBuilderView.tsx | 6 - .../components/overview/overviewData.test.ts | 24 + .../components/overview/overviewData.ts | 7 +- .../ModelDeprecationAlerts/AlertsInbox.tsx | 108 ++++ .../DeprecationAlertItem.tsx | 35 ++ .../DeprecationModel.tsx | 32 ++ .../MigrationDetail.tsx | 335 ++++++++++++ .../ModelAlertsPage.integration.test.tsx | 124 +++++ .../ModelDeprecationsPanel.tsx | 76 +++ .../ModelUsageChart.integration.test.tsx | 103 ++++ .../ModelUsageChart.tsx | 127 +++++ .../RecommendedReplacement.tsx | 7 + .../ModelDeprecationAlerts/UsageTags.test.tsx | 21 + .../ModelDeprecationAlerts/UsageTags.tsx | 40 ++ .../deprecationAlerts.test.ts | 78 +++ .../deprecationAlerts.ts | 61 +++ .../migrationData.test.ts | 43 ++ .../ModelDeprecationAlerts/migrationData.ts | 49 ++ .../ModelDeprecationAlerts/migrationRoute.ts | 14 + .../migrationState.test.ts | 37 ++ .../ModelDeprecationAlerts/migrationState.ts | 17 + .../useDeprecationAlerts.integration.test.tsx | 74 +++ .../useDeprecationAlerts.ts | 51 ++ .../NotificationsBell.test.tsx | 82 +++ .../NotificationsBell/NotificationsBell.tsx | 65 ++- .../src/components/leftnav.test.tsx | 1 + .../src/components/leftnav.tsx | 7 + .../model_info_view.integration.test.tsx | 79 +-- .../src/components/model_info_view.tsx | 3 +- .../src/components/page_metadata.ts | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 190 +++++++ 58 files changed, 2180 insertions(+), 1589 deletions(-) create mode 100644 litellm/proxy/alerting_endpoints/__init__.py create mode 100644 litellm/proxy/alerting_endpoints/model_deprecation_alerts.py create mode 100644 tests/unit/proxy/management_endpoints/test_model_deprecation_alerts.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/models/invalidateModelQueries.ts delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelDeprecations.test.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelDeprecations.ts delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.integration.test.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelDeprecationBanner.test.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelDeprecationBanner.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.integration.test.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/deprecationBannerText.test.ts delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/deprecationBannerText.ts create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationModel.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/MigrationDetail.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelDeprecationsPanel.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelUsageChart.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelUsageChart.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/RecommendedReplacement.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/UsageTags.test.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/UsageTags.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.test.ts create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.ts create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationData.test.ts create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationData.ts create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationRoute.ts create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationState.test.ts create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationState.ts create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts diff --git a/litellm/proxy/alerting_endpoints/__init__.py b/litellm/proxy/alerting_endpoints/__init__.py new file mode 100644 index 00000000000..6a7a204b435 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/__init__.py @@ -0,0 +1 @@ +"""HTTP endpoints for gateway alerts.""" diff --git a/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py new file mode 100644 index 00000000000..a69dd6ee025 --- /dev/null +++ b/litellm/proxy/alerting_endpoints/model_deprecation_alerts.py @@ -0,0 +1,109 @@ +"""Shared acknowledgement state for model migration alerts.""" + +import hashlib +import json +from datetime import date, datetime, timezone +from typing import Annotated, Final + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import TypeAdapter + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.deps.database import DatabaseDep +from litellm.repositories.config_repository import ConfigRepository +from litellm.repositories.daily_activity_buckets import api_key_bucket, api_key_from_bucket +from litellm.repositories.table_repositories import DailyTagSpendRepository +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.model_deprecation import DeprecationAlertState, DeprecationAlertUpdate, DeprecationTagUsage + +router: Final = APIRouter() + + +def alert_state_repository(database: DatabaseDep) -> ConfigRepository: + if database is None: + raise HTTPException(status_code=503, detail="Alert state requires a database connection") + return ConfigRepository(database.prisma, use_writer=True) + + +def alert_tag_repository(database: DatabaseDep) -> DailyTagSpendRepository: + if database is None: + raise HTTPException(status_code=503, detail="Model usage tags require a database connection") + return DailyTagSpendRepository(database.prisma) + + +def _state_key(model: str, deprecation_date: date) -> str: + identity: Final = json.dumps([model, deprecation_date.isoformat()]) + digest: Final = hashlib.sha256(identity.encode()).hexdigest() + return f"model_deprecation_alert:{digest}" + + +def _require_admin(user: UserAPIKeyAuth, *, write: bool) -> None: + if user.user_role == LitellmUserRoles.PROXY_ADMIN: + return + if not write and user.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: + return + raise HTTPException(status_code=403, detail="Only gateway admins can manage model deprecation alerts") + + +@router.get("/alerting/model-deprecations/state", tags=["alerting"], response_model=DeprecationAlertState) +async def get_deprecation_alert_state( + model: Annotated[str, Query(min_length=1, max_length=1024)], + deprecation_date: date, + user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[ConfigRepository, Depends(alert_state_repository)], +) -> DeprecationAlertState: + _require_admin(user, write=False) + record: Final = await repository.get_param(_state_key(model, deprecation_date)) + if record is None: + return DeprecationAlertState(model=model, deprecation_date=deprecation_date) + return DeprecationAlertState.model_validate(record.param_value) + + +@router.put("/alerting/model-deprecations/state", tags=["alerting"], response_model=DeprecationAlertState) +async def update_deprecation_alert_state( + data: DeprecationAlertUpdate, + user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[ConfigRepository, Depends(alert_state_repository)], +) -> DeprecationAlertState: + _require_admin(user, write=True) + state: Final = DeprecationAlertState( + **data.model_dump(), updated_at=datetime.now(timezone.utc), updated_by=user.user_id + ) + await repository.set_param(_state_key(data.model, data.deprecation_date), state.model_dump(mode="json")) + return state + + +@router.get("/alerting/model-deprecations/tags", tags=["alerting"], response_model=tuple[DeprecationTagUsage, ...]) +async def get_deprecation_usage_tags( + model: Annotated[str, Query(min_length=1, max_length=1024)], + api_keys: Annotated[tuple[str, ...], Query(min_length=1, max_length=100)], + start_date: date, + end_date: date, + user: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], + repository: Annotated[DailyTagSpendRepository, Depends(alert_tag_repository)], +) -> tuple[DeprecationTagUsage, ...]: + """Distinct tags observed with this model for a page of affected reporting key IDs.""" + _require_admin(user, write=False) + if not 0 <= (end_date - start_date).days < 30: + raise HTTPException(status_code=422, detail="Select up to 30 consecutive calendar dates") + rows: Final = await repository.table.group_by( + by=["api_key", "tag"], + where={ + "api_key": { + # bounded-ok: query validation limits api_keys to 100 IDs. + "in": [api_key_from_bucket(key) for key in api_keys] + }, + "model": model, + "date": {"gte": start_date.isoformat(), "lte": end_date.isoformat()}, + "api_requests": {"gt": 0}, + "tag": {"not": None}, + }, + ) + tags: Final = TypeAdapter(tuple[DeprecationTagUsage, ...]).validate_python(rows) + return tuple( + sorted( + (DeprecationTagUsage(api_key=api_key_bucket(usage.api_key), tag=usage.tag) for usage in tags), + key=lambda usage: (usage.api_key, usage.tag), + ) + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index fddecb3d6eb..9a80761b477 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -424,6 +424,9 @@ from litellm.messages.dispatch import messages_target from litellm.proxy._lazy_features import attach_lazy_features, reserve_lazy_slot from litellm.proxy._types import * from litellm.proxy.agent_endpoints.auth.agent_access_groups import CeilingResolver, resolve_agent_access_group_ceiling +from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( + router as model_deprecation_alerts_router, +) from litellm.proxy.analytics_endpoints.analytics_endpoints import ( router as analytics_router, ) @@ -22434,6 +22437,7 @@ app.include_router(prompt_caching_requests_router) app.include_router(router_settings_router) app.include_router(fallback_management_router) app.include_router(cache_settings_router) +app.include_router(model_deprecation_alerts_router) app.include_router(coordination_redis_settings_router) app.include_router(user_agent_analytics_router) app.include_router(gateway_request_router) diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index 7e459904d59..af21cc770de 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -1,9 +1,10 @@ from __future__ import annotations from datetime import date, datetime +from enum import Enum from typing import Final, Literal -from pydantic import Field +from pydantic import ConfigDict, Field from litellm.types.llms.base import LiteLLMBaseModel @@ -63,3 +64,38 @@ class ModelDeprecationResponse(LiteLLMBaseModel): ) warn_within_days: int = Field(description="The window (in days) used to bucket 'imminent' models.") checked_at: datetime = Field(description="UTC timestamp when the deprecation snapshot was generated.") + + +class DeprecationAlertStatus(str, Enum): + OPEN = "open" + RESOLVED = "resolved" + DISMISSED = "dismissed" + + +class DeprecationAlertUpdate(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str = Field(min_length=1, max_length=1024) + deprecation_date: date + status: DeprecationAlertStatus + last_seen_date: date + last_seen_requests: int = Field(ge=0) + + +class DeprecationAlertState(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + model: str + deprecation_date: date + status: DeprecationAlertStatus = DeprecationAlertStatus.OPEN + last_seen_date: date | None = None + last_seen_requests: int = 0 + updated_at: datetime | None = None + updated_by: str | None = None + + +class DeprecationTagUsage(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + api_key: str + tag: str diff --git a/tests/unit/proxy/management_endpoints/test_model_deprecation_alerts.py b/tests/unit/proxy/management_endpoints/test_model_deprecation_alerts.py new file mode 100644 index 00000000000..434dcc7a83c --- /dev/null +++ b/tests/unit/proxy/management_endpoints/test_model_deprecation_alerts.py @@ -0,0 +1,180 @@ +from collections.abc import Iterator + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.alerting_endpoints.model_deprecation_alerts import alert_state_repository, router +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.repositories.config_repository import ConfigParam, ConfigRepository +from litellm.types.proxy.auth.user_roles import LitellmUserRoles + + +class MemoryConfig(ConfigRepository): + def __init__(self) -> None: + self.rows: dict[str, object] = {} + + async def get_param(self, param_name: str) -> ConfigParam | None: + value = self.rows.get(param_name) + return None if value is None else ConfigParam(param_name, value) + + async def set_param(self, param_name: str, param_value: object) -> ConfigParam: + self.rows[param_name] = param_value + return ConfigParam(param_name, param_value) + + +@pytest.fixture() +def client() -> Iterator[tuple[TestClient, UserAPIKeyAuth]]: + app = FastAPI() + app.include_router(router) + user = UserAPIKeyAuth(user_id="admin-one", user_role=LitellmUserRoles.PROXY_ADMIN) + repository = MemoryConfig() + app.dependency_overrides[user_api_key_auth] = lambda: user + app.dependency_overrides[alert_state_repository] = lambda: repository + with TestClient(app) as http: + yield http, user + + +IDENTITY = {"model": "anthropic/claude-sonnet-4-5", "deprecation_date": "2026-11-30"} +UPDATE = {**IDENTITY, "status": "resolved", "last_seen_date": "2026-11-04", "last_seen_requests": 123} +PATH = "/alerting/model-deprecations/state" + + +def test_new_alert_is_open(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + http, _ = client + response = http.get(PATH, params=IDENTITY) + assert response.status_code == 200 + assert response.json()["status"] == "open" + + +def test_resolution_is_shared_and_keeps_the_usage_snapshot(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + http, user = client + response = http.put(PATH, json=UPDATE) + assert response.status_code == 200 + assert response.json()["updated_by"] == "admin-one" + user.user_id = "admin-two" + observed = http.get(PATH, params=IDENTITY).json() + assert observed["status"] == "resolved" + assert observed["last_seen_date"] == "2026-11-04" + assert observed["last_seen_requests"] == 123 + assert observed["updated_by"] == "admin-one" + + +def test_reopen_does_not_change_another_models_resolution(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + http, _ = client + http.put(PATH, json=UPDATE) + other = {**UPDATE, "model": "openai/gpt-4-turbo"} + http.put(PATH, json=other) + assert http.put(PATH, json={**UPDATE, "status": "open"}).json()["status"] == "open" + assert http.get(PATH, params={**IDENTITY, "model": other["model"]}).json()["status"] == "resolved" + + +@pytest.mark.parametrize("method", ["get", "put"]) +def test_internal_users_cannot_read_or_change_global_alerts( + client: tuple[TestClient, UserAPIKeyAuth], method: str +) -> None: + http, user = client + user.user_role = LitellmUserRoles.INTERNAL_USER + response = http.get(PATH, params=IDENTITY) if method == "get" else http.put(PATH, json=UPDATE) + assert response.status_code == 403 + + +def test_viewer_can_read_but_cannot_resolve_at_the_same_path(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + http, user = client + user.user_role = LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY + assert http.get(PATH, params=IDENTITY).status_code == 200 + assert http.put(PATH, json=UPDATE).status_code == 403 + assert http.get(PATH, params=IDENTITY).json()["status"] == "open" + + +@pytest.mark.parametrize( + "field,value", [("status", "unknown"), ("last_seen_requests", -1), ("deprecation_date", "invalid"), ("model", "")] +) +def test_invalid_updates_are_rejected(client: tuple[TestClient, UserAPIKeyAuth], field: str, value: object) -> None: + http, _ = client + assert http.put(PATH, json={**UPDATE, field: value}).status_code == 422 + assert http.get(PATH, params=IDENTITY).json()["status"] == "open" + + +def test_tags_are_scoped_to_model_dates_and_the_visible_keys(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.proxy.alerting_endpoints.model_deprecation_alerts import alert_tag_repository + + http, _ = client + grouped = AsyncMock(return_value=[{"api_key": "reporting-one", "tag": "support"}]) + http.app.dependency_overrides[alert_tag_repository] = lambda: SimpleNamespace( + table=SimpleNamespace(group_by=grouped) + ) + response = http.get( + "/alerting/model-deprecations/tags", + params={ + "model": "openai/gpt-4-turbo", + "api_keys": ["reporting-one", "reporting-two"], + "start_date": "2026-10-06", + "end_date": "2026-11-04", + }, + ) + assert response.status_code == 200 + assert response.json() == [{"api_key": "reporting-one", "tag": "support"}] + assert grouped.call_args.kwargs["where"] == { + "api_key": {"in": ["reporting-one", "reporting-two"]}, + "model": "openai/gpt-4-turbo", + "date": {"gte": "2026-10-06", "lte": "2026-11-04"}, + "api_requests": {"gt": 0}, + "tag": {"not": None}, + } + + +@pytest.mark.parametrize( + "role,expected", [(LitellmUserRoles.INTERNAL_USER, 403), (LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, 200)] +) +def test_tag_access_matches_alert_visibility( + client: tuple[TestClient, UserAPIKeyAuth], role: LitellmUserRoles, expected: int +) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.proxy.alerting_endpoints.model_deprecation_alerts import alert_tag_repository + + http, user = client + user.user_role = role + grouped = AsyncMock(return_value=[]) + http.app.dependency_overrides[alert_tag_repository] = lambda: SimpleNamespace( + table=SimpleNamespace(group_by=grouped) + ) + response = http.get( + "/alerting/model-deprecations/tags", + params={"model": "test-model", "api_keys": ["one"], "start_date": "2026-10-06", "end_date": "2026-11-04"}, + ) + assert response.status_code == expected + if expected == 403: + grouped.assert_not_called() + + +def test_tag_query_rejects_more_than_100_keys_before_database_access(client: tuple[TestClient, UserAPIKeyAuth]) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.proxy.alerting_endpoints.model_deprecation_alerts import ( + alert_tag_repository, + ) + + http, _ = client + grouped = AsyncMock(return_value=[]) + http.app.dependency_overrides[alert_tag_repository] = lambda: SimpleNamespace( + table=SimpleNamespace(group_by=grouped) + ) + response = http.get( + "/alerting/model-deprecations/tags", + params={ + "model": "test-model", + "api_keys": [f"reporting-{index}" for index in range(101)], + "start_date": "2026-10-07", + "end_date": "2026-11-05", + }, + ) + assert response.status_code == 422 + grouped.assert_not_called() diff --git a/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx new file mode 100644 index 00000000000..ec88a69670e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/alerts/page.tsx @@ -0,0 +1,19 @@ +"use client"; + +import { Page, PageTabs, PageTabsList, PageTabsTrigger, PageTabsContent } from "@/components/shared/Page"; +import ModelDeprecationsPanel from "@/components/ModelDeprecationAlerts/ModelDeprecationsPanel"; + +export default function AlertsPage() { + return ( + + + + Model deprecations + + + + + + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/invalidateModelQueries.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/invalidateModelQueries.ts deleted file mode 100644 index 3382d6c9c81..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/invalidateModelQueries.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { QueryClient } from "@tanstack/react-query"; -import { modelDeprecationKeys } from "./useModelDeprecations"; - -export const invalidateModelQueries = async (queryClient: QueryClient): Promise => { - await Promise.all([ - queryClient.invalidateQueries({ queryKey: ["models", "list"] }), - queryClient.invalidateQueries({ queryKey: modelDeprecationKeys.all }), - ]); -}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelDeprecations.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelDeprecations.test.tsx deleted file mode 100644 index 88efd721c26..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelDeprecations.test.tsx +++ /dev/null @@ -1,64 +0,0 @@ -/* @vitest-environment jsdom */ -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { act, renderHook, waitFor } from "@testing-library/react"; -import type { ReactNode } from "react"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { useModelDeprecations } from "./useModelDeprecations"; - -const authState = vi.hoisted(() => ({ accessToken: "token", userRole: "internal_user" })); -const emptyDeprecationsResponse = { - checked_at: "2026-11-04T00:00:00Z", - warn_within_days: 30, - deprecated: [], - imminent: [], - upcoming: [], -}; -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: () => ({ accessToken: authState.accessToken, userRole: authState.userRole }), -})); - -describe("useModelDeprecations", () => { - let fetchMock: ReturnType>; - - beforeEach(() => { - authState.accessToken = "token"; - authState.userRole = "internal_user"; - fetchMock = vi.fn().mockResolvedValue(Response.json(emptyDeprecationsResponse)); - vi.stubGlobal("fetch", fetchMock); - }); - - afterEach(() => { - vi.unstubAllGlobals(); - }); - - it("does not fetch deprecations for an internal user", async () => { - const queryClient = new QueryClient(); - const wrapper = ({ children }: { children: ReactNode }) => ( - {children} - ); - - const { result } = renderHook(() => useModelDeprecations(), { wrapper }); - - await waitFor(() => expect(result.current.fetchStatus).toBe("idle")); - await act(async () => { - await new Promise((resolve) => setTimeout(resolve, 0)); - }); - expect(fetchMock).not.toHaveBeenCalled(); - }); - - it("fetches deprecations for a proxy admin viewer", async () => { - authState.userRole = "proxy_admin_viewer"; - const queryClient = new QueryClient(); - const wrapper = ({ children }: { children: ReactNode }) => ( - {children} - ); - - renderHook(() => useModelDeprecations(), { wrapper }); - - await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1)); - const request = fetchMock.mock.calls[0][0]; - expect(new URL(request instanceof Request ? request.url : String(request), "http://localhost").pathname).toBe( - "/model/deprecations", - ); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelDeprecations.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelDeprecations.ts deleted file mode 100644 index bb9922d8a8b..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelDeprecations.ts +++ /dev/null @@ -1,33 +0,0 @@ -"use client"; - -import { useQuery } from "@tanstack/react-query"; -import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { backgroundFetchClient } from "@/lib/http/api"; -import type { components } from "@/lib/http/schema"; -import { isProxyAdminTierRole } from "@/utils/roles"; -import { createQueryKeys } from "../common/queryKeysFactory"; - -export type ModelDeprecationInfo = components["schemas"]["ModelDeprecationInfo"]; -export type ModelDeprecationResponse = components["schemas"]["ModelDeprecationResponse"]; - -export const modelDeprecationKeys = createQueryKeys("modelDeprecations"); - -const fetchModelDeprecations = async (): Promise => { - const { data } = await backgroundFetchClient.GET("/model/deprecations"); - if (data === undefined) { - throw new Error("The proxy returned no model deprecations payload"); - } - return data; -}; - -export const useModelDeprecations = () => { - const { accessToken, userRole } = useAuthorized(); - const queryOptions = { - queryKey: modelDeprecationKeys.all, - queryFn: fetchModelDeprecations, - enabled: Boolean(accessToken) && isProxyAdminTierRole(userRole), - staleTime: 5 * 60 * 1000, - retry: false, - }; - return useQuery(queryOptions); -}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts index 7e9669e6ce5..f9304bab0d2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModels.ts @@ -1,7 +1,6 @@ import { useQuery, useInfiniteQuery, useQueryClient, UseQueryResult } from "@tanstack/react-query"; import { useMemo } from "react"; import { createQueryKeys } from "../common/queryKeysFactory"; -import { invalidateModelQueries } from "./invalidateModelQueries"; import { modelInfoCall, modelHubCall, modelAvailableCall } from "@/components/networking"; import useAuthorized from "../useAuthorized"; import { EndpointType, isModeCompatibleWithEndpoint } from "@/components/chat_ui/mode_endpoint_mapping"; @@ -241,7 +240,9 @@ export const useAutoRouters = (): UseQueryResult export const useInvalidateAutoRouters = (): (() => Promise) => { const queryClient = useQueryClient(); - return () => invalidateModelQueries(queryClient); + return async () => { + await queryClient.invalidateQueries({ queryKey: modelKeys.lists() }); + }; }; export const useModelHub = () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts index fe054c7b39a..19dac805948 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/legacyPageRoutes.ts @@ -31,6 +31,7 @@ const LEGACY_PAGE_ROUTES: ReadonlyMap = new Map( "ui-theme": "ui-theme", telemetry: "telemetry", logs: "logs", + alerts: "alerts", lens: "lens", "admin-panel": "admin-panel", "logging-and-alerts": "logging-and-alerts", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.integration.test.tsx deleted file mode 100644 index 42747bfeeca..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.integration.test.tsx +++ /dev/null @@ -1,232 +0,0 @@ -import * as useAuthorizedModule from "@/app/(dashboard)/hooks/useAuthorized"; -import { useQuery } from "@tanstack/react-query"; -import { screen, waitFor } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - -import { renderWithProviders, testQueryClient } from "../../../../../tests/test-utils"; -import AllModelsTab from "./AllModelsTab"; -import ModelDeprecationBanner from "./ModelDeprecationBanner"; - -const mockModelDeleteCall = vi.fn().mockResolvedValue({}); -const mockModelPatchUpdateCall = vi.fn().mockResolvedValue({}); -vi.mock("@/components/networking", () => ({ - serverRootPath: "/", - modelDeleteCall: (...args: unknown[]) => mockModelDeleteCall(...args), - modelPatchUpdateCall: (...args: unknown[]) => mockModelPatchUpdateCall(...args), -})); - -vi.mock("@/components/model_dashboard/ModelSettingsModal/ModelSettingsModal", () => ({ - default: function ModelSettingsModalMock({ isVisible }: { isVisible: boolean }) { - return isVisible ?
: null; - }, -})); - -let modelsInfoResult: Record = {}; -const mockRefetch = vi.fn(); -vi.mock("../../hooks/models/useModels", () => ({ - useModelsInfo: () => ({ ...modelsInfoResult, refetch: mockRefetch }), -})); - -vi.mock("../../hooks/models/useModelCostMap", () => ({ - useModelCostMap: () => ({ data: { "gpt-4": { litellm_provider: "openai" } }, isLoading: false, error: null }), -})); - -const mockTeams = [{ team_id: "team-1", team_alias: "Engineering" }]; -const mockCredentials = [ - { credential_name: "openai-prod", display_name: "Prod OpenAI", credential_values: {}, credential_info: {} }, -]; -const mockUseCredentials = vi.hoisted(() => vi.fn()); -vi.mock("../../hooks/credentials/useCredentials", () => ({ - useCredentials: mockUseCredentials, -})); - -vi.mock("../../hooks/teams/useTeams", () => ({ - useTeams: () => ({ data: mockTeams, isLoading: false, error: null, refetch: vi.fn() }), -})); - -const BASE_MODEL_INFO = { - id: "model-1", - db_model: true, - created_by: "user-123", - created_at: "2024-01-01T00:00:00Z", - updated_at: "2024-01-02T00:00:00Z", - team_id: "team-1", - access_groups: [], -}; - -const makeRow = () => ({ - model_name: "gpt-4", - litellm_params: { model: "openai/gpt-4", custom_llm_provider: "openai" }, - model_info: BASE_MODEL_INFO, -}); - -const setModelsInfo = (rows: Record[]) => { - modelsInfoResult = { - data: { data: rows, total_count: rows.length, current_page: 1, total_pages: 1, size: 50 }, - isLoading: false, - isFetching: false, - error: null, - }; -}; - -const MOCK_AUTHORIZED = { - isLoading: false, - isAuthorized: true, - token: "mock-token", - accessToken: "mock-access-token", - userId: "user-123", - userEmail: "test@example.com", - userRole: "Admin", - userRoleLabel: "Admin", - isViewOnly: false, - premiumUser: true, - disabledPersonalKeyCreation: false, - showSSOBanner: false, -}; - -const mockSetSelectedModelGroup = vi.fn(); -const mockSetSelectedModelId = vi.fn(); -const mockSetSelectedTeamId = vi.fn(); - -const defaultProps = { - selectedModelGroup: "all", - setSelectedModelGroup: mockSetSelectedModelGroup, - availableModelGroups: ["gpt-4", "gpt-3.5-turbo"], - availableModelAccessGroups: ["sales-team"], - setSelectedModelId: mockSetSelectedModelId, - setSelectedTeamId: mockSetSelectedTeamId, -}; - -const TEST_TODAY = new Date(2026, 10, 4); - -const deprecatedResponse = { - checked_at: "2026-06-01T00:00:00Z", - warn_within_days: 30, - deprecated: [ - { - model_name: "old-model", - litellm_provider: null, - deprecation_date: "2026-06-01", - days_until_deprecation: -1, - status: "deprecated", - successor_model: null, - }, - ], - imminent: [], - upcoming: [], -}; - -const currentResponse = { - checked_at: "2026-06-02T00:00:00Z", - warn_within_days: 30, - deprecated: [], - imminent: [ - { - model_name: "current-model", - litellm_provider: null, - deprecation_date: "2026-11-30", - days_until_deprecation: 26, - status: "imminent", - successor_model: null, - }, - ], - upcoming: [], -}; - -const stubDeprecationResponses = () => { - const fetchMock = vi - .fn() - .mockImplementationOnce(async (input) => { - const requestUrl = input instanceof Request ? input.url : String(input); - expect(new URL(requestUrl, "http://localhost").pathname).toBe("/model/deprecations"); - return Response.json(deprecatedResponse); - }) - .mockImplementationOnce(async (input) => { - const requestUrl = input instanceof Request ? input.url : String(input); - expect(new URL(requestUrl, "http://localhost").pathname).toBe("/model/deprecations"); - return Response.json(currentResponse); - }); - vi.stubGlobal("fetch", fetchMock); - return fetchMock; -}; - -const renderModelsWithDeprecationBanner = () => - renderWithProviders( - <> - - - , - ); - -const ModelListQueryProbe = ({ queryFn }: { queryFn: () => Promise }) => { - useQuery({ queryKey: ["models", "list"], queryFn, staleTime: Infinity }); - return null; -}; - -describe("AllModelsTab", () => { - beforeEach(() => { - vi.clearAllMocks(); - testQueryClient.clear(); - setModelsInfo([makeRow()]); - vi.spyOn(useAuthorizedModule, "default").mockReturnValue(MOCK_AUTHORIZED); - mockUseCredentials.mockImplementation(({ enabled = true }: { enabled?: boolean } = {}) => ({ - data: enabled ? { credentials: mockCredentials } : undefined, - isLoading: false, - })); - }); - - afterEach(() => { - vi.unstubAllGlobals(); - }); - - it("refreshes deprecation banner data after deleting a model", async () => { - const fetchMock = stubDeprecationResponses(); - const user = userEvent.setup(); - renderModelsWithDeprecationBanner(); - - expect(await screen.findByText("old-model was deprecated on 2026-06-01")).toBeInTheDocument(); - await user.click(await screen.findByTestId("model-delete-model-1")); - await user.click(await screen.findByRole("button", { name: /^delete$/i })); - expect(await screen.findByText("current-model deprecates on 2026-11-30, in 26 days")).toBeInTheDocument(); - await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); - expect(screen.queryByText("old-model was deprecated on 2026-06-01")).not.toBeInTheDocument(); - }); - - it("refreshes deprecation banner data after pausing a model", async () => { - const fetchMock = stubDeprecationResponses(); - const user = userEvent.setup(); - renderModelsWithDeprecationBanner(); - - expect(await screen.findByText("old-model was deprecated on 2026-06-01")).toBeInTheDocument(); - await user.click(await screen.findByTestId("model-pause-toggle-model-1")); - expect(await screen.findByText("current-model deprecates on 2026-11-30, in 26 days")).toBeInTheDocument(); - await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); - expect(screen.queryByText("old-model was deprecated on 2026-06-01")).not.toBeInTheDocument(); - }); - - it("refreshes the table and deprecations once without refetching another models list observer", async () => { - const fetchMock = stubDeprecationResponses(); - const observerQueryFn = vi.fn<() => Promise>().mockResolvedValue("models"); - const user = userEvent.setup(); - renderWithProviders( - <> - - - - , - ); - - expect(await screen.findByText("old-model was deprecated on 2026-06-01")).toBeInTheDocument(); - await waitFor(() => expect(observerQueryFn).toHaveBeenCalledTimes(1)); - mockRefetch.mockClear(); - fetchMock.mockClear(); - - await user.click(screen.getByTestId("datatable-refresh")); - - expect(await screen.findByText("current-model deprecates on 2026-11-30, in 26 days")).toBeInTheDocument(); - await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1)); - expect(observerQueryFn).toHaveBeenCalledTimes(1); - expect(mockRefetch).toHaveBeenCalledTimes(1); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx index 97e7f0238bb..09e00db9fb0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.test.tsx @@ -4,7 +4,7 @@ import userEvent from "@testing-library/user-event"; import type { OnUrlUpdateFunction } from "nuqs/adapters/testing"; import { beforeEach, describe, expect, it, Mock, vi } from "vitest"; -import { renderWithProviders, testQueryClient } from "../../../../../tests/test-utils"; +import { renderWithProviders } from "../../../../../tests/test-utils"; import AllModelsTab from "./AllModelsTab"; import { STATUS_COLUMN_ID, toServerSortField } from "./ModelsTableColumns"; @@ -22,6 +22,12 @@ vi.mock("@/components/model_dashboard/ModelSettingsModal/ModelSettingsModal", () }, })); +const mockInvalidateQueries = vi.fn(); +vi.mock("@tanstack/react-query", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, useQueryClient: () => ({ invalidateQueries: mockInvalidateQueries }) }; +}); + interface ModelsInfoArgs { page?: number; size?: number; @@ -151,7 +157,6 @@ const defaultProps = { describe("AllModelsTab", () => { beforeEach(() => { vi.clearAllMocks(); - testQueryClient.clear(); modelsInfoCalls.length = 0; setModelsInfo([makeRow()]); vi.spyOn(useAuthorizedModule, "default").mockReturnValue(MOCK_AUTHORIZED); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx index cf977a6d9e3..ff027ac8b3c 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/AllModelsTab.tsx @@ -22,8 +22,6 @@ import { useCallback, useMemo, useState } from "react"; import { Trans, useTranslation } from "react-i18next"; import { useModelsInfo } from "../../hooks/models/useModels"; -import { modelDeprecationKeys } from "../../hooks/models/useModelDeprecations"; -import { invalidateModelQueries } from "../../hooks/models/invalidateModelQueries"; import { ROUTING_STATUS_FILTER_ID, routingStatusToBlocked } from "../utils/routingStatus"; import { transformModelData } from "../utils/modelDataTransformer"; import { @@ -268,8 +266,8 @@ const AllModelsTab = ({ try { setDeleteLoading(true); await modelDeleteCall(accessToken, deleteModalModelId); - void invalidateModelQueries(queryClient); toast.success(t("modelsAndEndpoints.allModelsTab.toasts.deleted")); + queryClient.invalidateQueries({ queryKey: ["models", "list"] }); refetchModels(); } catch (error) { console.error("Error deleting model:", error); @@ -293,7 +291,7 @@ const AllModelsTab = ({ ); // invalidateQueries already schedules a refetch for active observers // on this key — no need to also call refetchModels() (would double-fetch). - void invalidateModelQueries(queryClient); + queryClient.invalidateQueries({ queryKey: ["models", "list"] }); } catch (error) { console.error("Error toggling model pause state:", error); toast.fromError(error); @@ -305,9 +303,8 @@ const AllModelsTab = ({ ); const handleRefresh = useCallback(() => { - refetchModels(); - void queryClient.invalidateQueries({ queryKey: modelDeprecationKeys.all }); - }, [queryClient, refetchModels]); + void refetchModels(); + }, [refetchModels]); const handleDeleteClick = useCallback((modelId: string) => { setDeleteModalModelId(modelId); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelDeprecationBanner.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelDeprecationBanner.test.tsx deleted file mode 100644 index c14bf93c116..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelDeprecationBanner.test.tsx +++ /dev/null @@ -1,477 +0,0 @@ -/* @vitest-environment jsdom */ -import { render, screen, within } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { beforeEach, describe, expect, it, vi } from "vitest"; -import { - useModelDeprecations, - type ModelDeprecationInfo, - type ModelDeprecationResponse, -} from "@/app/(dashboard)/hooks/models/useModelDeprecations"; -import { uiHref } from "@/utils/uiHref"; -import ModelDeprecationBanner from "./ModelDeprecationBanner"; - -vi.mock("@/app/(dashboard)/hooks/models/useModelDeprecations", () => ({ - useModelDeprecations: vi.fn(), -})); - -const mockedUseModelDeprecations = vi.mocked(useModelDeprecations); - -const makeResponse = (overrides: Partial = {}): ModelDeprecationResponse => ({ - checked_at: "2026-11-04T00:00:00Z", - warn_within_days: 30, - deprecated: [], - imminent: [], - upcoming: [], - ...overrides, -}); - -const makeInfo = (overrides: Partial = {}): ModelDeprecationInfo => ({ - model_name: "gpt-4o", - deprecation_date: "2026-11-30", - days_until_deprecation: 26, - status: "imminent", - ...overrides, -}); - -const queryResult = ( - data: ModelDeprecationResponse | undefined, - isLoading = false, - isError = false, - overrides: Partial> = {}, -): ReturnType => - ({ data, isLoading, isError, ...overrides }) as ReturnType; - -const pinnedToday = (): Date => new Date(2026, 10, 4); - -const renderBanner = (today = pinnedToday()) => render(); - -describe("ModelDeprecationBanner", () => { - beforeEach(() => { - vi.clearAllMocks(); - window.localStorage.clear(); - }); - - it("renders nothing when there are no deprecated or imminent models", () => { - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse())); - - renderBanner(); - - expect(screen.queryByRole("alert")).not.toBeInTheDocument(); - }); - - it("renders nothing when only upcoming models are present", () => { - mockedUseModelDeprecations.mockReturnValue( - queryResult( - makeResponse({ - upcoming: [makeInfo({ deprecation_date: "2026-12-05", status: "upcoming" })], - }), - ), - ); - - renderBanner(); - - expect(screen.queryByRole("alert")).not.toBeInTheDocument(); - }); - - it("includes server-upcoming models at the local warning-window boundary", () => { - const upcomingInfoOverrides: Partial = { - model_name: "due-in-window", - deprecation_date: "2026-12-04", - days_until_deprecation: 31, - status: "upcoming", - }; - const info = makeInfo(upcomingInfoOverrides); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ upcoming: [info] }))); - - renderBanner(); - - expect(screen.getByText("1 model deprecates within 30 days")).toBeInTheDocument(); - expect(within(screen.getByRole("list")).getByRole("listitem")).toHaveTextContent( - "due-in-window deprecates on 2026-12-04, in 30 days", - ); - }); - - it("does not show server-upcoming models one day past the local warning window", () => { - const upcomingInfoOverrides: Partial = { - model_name: "due-outside-window", - deprecation_date: "2026-12-05", - days_until_deprecation: 31, - status: "upcoming", - }; - const info = makeInfo(upcomingInfoOverrides); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ upcoming: [info] }))); - - renderBanner(); - - expect(screen.queryByRole("alert")).not.toBeInTheDocument(); - }); - - it("renders nothing while loading or after an error", () => { - mockedUseModelDeprecations.mockReturnValue(queryResult(undefined, true)); - const { rerender } = renderBanner(); - expect(screen.queryByRole("alert")).not.toBeInTheDocument(); - - mockedUseModelDeprecations.mockReturnValue(queryResult(undefined, false, true)); - rerender(); - - expect(screen.queryByRole("alert")).not.toBeInTheDocument(); - }); - - it("keeps the banner visible when a refetch fails", () => { - const staleInfoOverrides: Partial = { - model_name: "stale-model", - deprecation_date: "2026-06-01", - days_until_deprecation: -156, - status: "deprecated", - }; - const info = makeInfo(staleInfoOverrides); - const refetchError = new Error("Refetch failed"); - const refetchErrorResult: Partial> = { - error: refetchError, - failureCount: 1, - failureReason: refetchError, - fetchStatus: "idle", - isFetching: false, - isLoadingError: false, - isRefetchError: true, - status: "error", - }; - mockedUseModelDeprecations.mockReturnValue( - queryResult(makeResponse({ deprecated: [info] }), false, true, refetchErrorResult), - ); - - renderBanner(); - - expect(screen.getByRole("alert")).toBeInTheDocument(); - expect(within(screen.getByRole("list")).getByRole("listitem")).toHaveTextContent( - "stale-model was deprecated on 2026-06-01", - ); - }); - - it("shows deprecated and imminent entries with a link to Logging & Alerts", () => { - const deprecatedInfo: ModelDeprecationInfo = { - model_name: "gpt-4", - team_public_model_name: null, - litellm_provider: "openai", - deprecation_date: "2026-06-01", - days_until_deprecation: -152, - status: "deprecated", - successor_model: "gpt-5.6-sol", - }; - const imminentInfo: ModelDeprecationInfo = { - model_name: "gpt-4o", - team_public_model_name: null, - litellm_provider: "openai", - deprecation_date: "2026-11-30", - days_until_deprecation: 26, - status: "imminent", - successor_model: "X", - }; - const response = makeResponse({ - deprecated: [deprecatedInfo], - imminent: [imminentInfo], - }); - mockedUseModelDeprecations.mockReturnValue(queryResult(response)); - - renderBanner(); - - expect(screen.getByText("1 model is deprecated, 1 deprecates within 30 days")).toBeInTheDocument(); - const entries = within(screen.getByRole("list")).getAllByRole("listitem"); - expect(entries[0]).toHaveTextContent("gpt-4 (openai) was deprecated on 2026-06-01, migrate to gpt-5.6-sol"); - expect(entries[1]).toHaveTextContent("gpt-4o (openai) deprecates on 2026-11-30, in 26 days, migrate to X"); - expect(screen.getByRole("link", { name: "Get Slack alerts for deprecations" })).toHaveAttribute( - "href", - `${uiHref("logging-and-alerts")}?tab=alerting-types`, - ); - }); - - it("treats a server-deprecated entry dated today as imminent locally", () => { - const infoOverrides: Partial = { - model_name: "due-today", - deprecation_date: "2026-11-04", - days_until_deprecation: -1, - status: "deprecated", - }; - const info = makeInfo(infoOverrides); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ deprecated: [info] }))); - - renderBanner(); - - expect(screen.getByText("1 model deprecates within 30 days")).toBeInTheDocument(); - expect(within(screen.getByRole("list")).getByRole("listitem")).toHaveTextContent( - "due-today deprecates today (2026-11-04)", - ); - }); - - it("treats a server-imminent entry dated yesterday as deprecated locally", () => { - const infoOverrides: Partial = { - model_name: "due-yesterday", - deprecation_date: "2026-11-03", - days_until_deprecation: 0, - status: "imminent", - }; - const info = makeInfo(infoOverrides); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ imminent: [info] }))); - - renderBanner(); - - expect(screen.getByText("1 model is deprecated")).toBeInTheDocument(); - expect(within(screen.getByRole("list")).getByRole("listitem")).toHaveTextContent( - "due-yesterday was deprecated on 2026-11-03", - ); - }); - - it("uses the local date difference instead of the server day count", () => { - const infoOverrides: Partial = { - model_name: "due-in-two", - deprecation_date: "2026-11-06", - days_until_deprecation: 1, - status: "imminent", - }; - const info = makeInfo(infoOverrides); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ imminent: [info] }))); - - renderBanner(); - - expect(within(screen.getByRole("list")).getByRole("listitem")).toHaveTextContent( - "due-in-two deprecates on 2026-11-06, in 2 days", - ); - }); - - it("deduplicates entries by their displayed name, provider, and deprecation date", () => { - const globalInfoOverrides: Partial = { - model_name: "o4-mini", - team_public_model_name: null, - litellm_provider: "openai", - deprecation_date: "2026-10-23", - status: "deprecated", - }; - const firstTeamInfoOverrides: Partial = { - model_name: "o4-mini-team-a", - team_public_model_name: "o4-mini", - litellm_provider: "openai", - deprecation_date: "2026-10-23", - status: "deprecated", - }; - const secondTeamInfoOverrides: Partial = { - model_name: "o4-mini-team-b", - team_public_model_name: "o4-mini", - litellm_provider: "openai", - deprecation_date: "2026-10-23", - status: "deprecated", - }; - mockedUseModelDeprecations.mockReturnValue( - queryResult( - makeResponse({ - deprecated: [ - makeInfo(globalInfoOverrides), - makeInfo(firstTeamInfoOverrides), - makeInfo(secondTeamInfoOverrides), - ], - }), - ), - ); - - renderBanner(); - - expect(screen.getByText("1 model is deprecated")).toBeInTheDocument(); - const entries = within(screen.getByRole("list")).getAllByRole("listitem"); - expect(entries).toHaveLength(1); - expect(entries[0]).toHaveTextContent("o4-mini (openai) was deprecated on 2026-10-23"); - }); - - it("keeps entries with different successors and renders each migration target", () => { - const firstEntryOverrides: Partial = { - model_name: "team-model-a", - team_public_model_name: "shared-model", - litellm_provider: "openai", - deprecation_date: "2026-10-23", - days_until_deprecation: -1, - status: "deprecated", - successor_model: "successor-a", - }; - const secondEntryOverrides: Partial = { - model_name: "shared-model", - litellm_provider: "openai", - deprecation_date: "2026-10-23", - days_until_deprecation: -1, - status: "deprecated", - successor_model: "successor-b", - }; - const firstEntry = makeInfo(firstEntryOverrides); - const secondEntry = makeInfo(secondEntryOverrides); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ deprecated: [firstEntry, secondEntry] }))); - - renderBanner(); - - const entries = within(screen.getByRole("list")).getAllByRole("listitem"); - expect(screen.getByText("1 model is deprecated")).toBeInTheDocument(); - expect(entries).toHaveLength(2); - expect(entries[0]).toHaveTextContent("shared-model (openai) was deprecated on 2026-10-23, migrate to successor-a"); - expect(entries[1]).toHaveTextContent("shared-model (openai) was deprecated on 2026-10-23, migrate to successor-b"); - }); - - it("keeps entries with different deprecation dates", () => { - const firstEntryOverrides: Partial = { - model_name: "shared-model", - litellm_provider: "openai", - deprecation_date: "2026-10-22", - days_until_deprecation: -2, - status: "deprecated", - successor_model: "successor", - }; - const secondEntryOverrides: Partial = { - model_name: "shared-model", - litellm_provider: "openai", - deprecation_date: "2026-10-23", - days_until_deprecation: -1, - status: "deprecated", - successor_model: "successor", - }; - const firstEntry = makeInfo(firstEntryOverrides); - const secondEntry = makeInfo(secondEntryOverrides); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ deprecated: [firstEntry, secondEntry] }))); - - renderBanner(); - - const entries = within(screen.getByRole("list")).getAllByRole("listitem"); - expect(entries).toHaveLength(2); - expect(entries[0]).toHaveTextContent("shared-model (openai) was deprecated on 2026-10-22"); - expect(entries[1]).toHaveTextContent("shared-model (openai) was deprecated on 2026-10-23"); - }); - - it("dismisses the banner and keeps it dismissed after remounting the same entries", async () => { - const info = makeInfo(); - const secondInfo = makeInfo({ model_name: "claude-sonnet-4" }); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ imminent: [info, secondInfo] }))); - const user = userEvent.setup(); - const { unmount } = renderBanner(); - - await user.click(screen.getByRole("button", { name: "Dismiss deprecation warning" })); - expect(screen.queryByRole("alert")).not.toBeInTheDocument(); - unmount(); - - const changedSecondInfoOverrides: Partial = { - ...secondInfo, - days_until_deprecation: secondInfo.days_until_deprecation - 1, - }; - const changedInfoOverrides: Partial = { - ...info, - days_until_deprecation: info.days_until_deprecation - 1, - }; - const teamAliasInfoOverrides: Partial = { - ...info, - model_name: "team-internal-name", - team_public_model_name: info.model_name, - days_until_deprecation: info.days_until_deprecation - 1, - }; - mockedUseModelDeprecations.mockReturnValue( - queryResult( - makeResponse({ - imminent: [ - makeInfo(changedSecondInfoOverrides), - makeInfo(changedInfoOverrides), - makeInfo(teamAliasInfoOverrides), - ], - }), - ), - ); - renderBanner(); - - expect(screen.queryByRole("alert")).not.toBeInTheDocument(); - }); - - it("shows the banner again when a new entry is added after dismissal", async () => { - const firstEntryOverrides: Partial = { - model_name: "gpt-4", - deprecation_date: "2026-10-23", - days_until_deprecation: -12, - status: "deprecated", - }; - const firstEntry = makeInfo(firstEntryOverrides); - const addedEntryOverrides: Partial = { - model_name: "gpt-3.5", - deprecation_date: "2026-10-23", - days_until_deprecation: -12, - status: "deprecated", - }; - const addedEntry = makeInfo(addedEntryOverrides); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ deprecated: [firstEntry] }))); - const user = userEvent.setup(); - const { rerender } = renderBanner(); - - await user.click(screen.getByRole("button", { name: "Dismiss deprecation warning" })); - mockedUseModelDeprecations.mockReturnValue( - queryResult( - makeResponse({ - deprecated: [firstEntry, addedEntry], - }), - ), - ); - rerender(); - - expect(screen.getByRole("alert")).toBeInTheDocument(); - expect(screen.getByText("2 models are deprecated")).toBeInTheDocument(); - }); - - it("shows the banner again when an imminent entry becomes deprecated", async () => { - const imminentEntryOverrides: Partial = { - model_name: "gpt-4o", - deprecation_date: "2026-11-30", - days_until_deprecation: 0, - status: "imminent", - }; - const imminentEntry = makeInfo(imminentEntryOverrides); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ imminent: [imminentEntry] }))); - const user = userEvent.setup(); - const { rerender } = renderBanner(new Date(2026, 10, 30)); - - await user.click(screen.getByRole("button", { name: "Dismiss deprecation warning" })); - rerender(); - - expect(screen.getByRole("alert")).toBeInTheDocument(); - expect(screen.getByText("1 model is deprecated")).toBeInTheDocument(); - }); - - it("shows the banner again when an entry's successor changes after dismissal", async () => { - const entry = makeInfo({ successor_model: "successor-a" }); - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ imminent: [entry] }))); - const user = userEvent.setup(); - const { rerender } = renderBanner(); - - await user.click(screen.getByRole("button", { name: "Dismiss deprecation warning" })); - expect(screen.queryByRole("alert")).not.toBeInTheDocument(); - - mockedUseModelDeprecations.mockReturnValue( - queryResult(makeResponse({ imminent: [makeInfo({ ...entry, successor_model: "successor-b" })] })), - ); - rerender(); - - expect(screen.getByRole("alert")).toBeInTheDocument(); - expect(screen.getByText(/migrate to successor-b/)).toBeInTheDocument(); - }); - - it("expands all entries and collapses back to the first five", async () => { - const imminent = [ - ...Array.from({ length: 7 }, (_value, index) => makeInfo({ model_name: `model-${index + 1}` })), - makeInfo({ model_name: "team-model-1", team_public_model_name: "model-1" }), - ]; - mockedUseModelDeprecations.mockReturnValue(queryResult(makeResponse({ imminent }))); - const user = userEvent.setup(); - - renderBanner(); - - const entries = within(screen.getByRole("list")).getAllByRole("listitem"); - expect(entries).toHaveLength(5); - expect(entries[4]).toHaveTextContent("model-5 deprecates"); - expect(screen.queryByText(/model-6/)).not.toBeInTheDocument(); - - await user.click(screen.getByRole("button", { name: "Show all 7" })); - expect(within(screen.getByRole("list")).getAllByRole("listitem")).toHaveLength(7); - expect(screen.getByText(/model-6 deprecates/)).toBeInTheDocument(); - - await user.click(screen.getByRole("button", { name: "Show fewer" })); - expect(within(screen.getByRole("list")).getAllByRole("listitem")).toHaveLength(5); - expect(screen.queryByText(/model-6/)).not.toBeInTheDocument(); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelDeprecationBanner.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelDeprecationBanner.tsx deleted file mode 100644 index 3a89940f786..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/ModelDeprecationBanner.tsx +++ /dev/null @@ -1,98 +0,0 @@ -"use client"; - -import { TriangleAlert, X } from "lucide-react"; -import { useState } from "react"; -import { useModelDeprecations } from "@/app/(dashboard)/hooks/models/useModelDeprecations"; -import { Alert, AlertAction, AlertDescription, AlertTitle } from "@/components/ui/alert"; -import { Button } from "@/components/ui/button"; -import { getLocalStorageItem, setLocalStorageItem } from "@/utils/localStorageUtils"; -import { uiHref } from "@/utils/uiHref"; -import { - bucketByLocalDate, - countDistinctModels, - dedupeDeprecationEntries, - deprecationBannerFingerprint, - deprecationBannerTitle, - deprecationEntryText, -} from "./deprecationBannerText"; - -const STORAGE_KEY = "modelDeprecationBannerDismissed"; - -const ModelDeprecationBanner = ({ today = new Date() }: { today?: Date }) => { - const { data } = useModelDeprecations(); - const [isExpanded, setIsExpanded] = useState(false); - const [dismissedFingerprint, setDismissedFingerprint] = useState(() => - getLocalStorageItem(STORAGE_KEY), - ); - - if (!data) { - return null; - } - - const localBuckets = bucketByLocalDate( - [...(data.deprecated ?? []), ...(data.imminent ?? []), ...(data.upcoming ?? [])], - today, - data.warn_within_days, - ); - const deprecated = dedupeDeprecationEntries(localBuckets.deprecated); - const imminent = dedupeDeprecationEntries(localBuckets.imminent); - - if (deprecated.length === 0 && imminent.length === 0) { - return null; - } - - const entries = [...deprecated, ...imminent]; - const fingerprint = deprecationBannerFingerprint(entries); - if (dismissedFingerprint === fingerprint) { - return null; - } - - const visibleEntries = isExpanded ? entries : entries.slice(0, 5); - - return ( - 0 ? "destructive" : "default"} className="mb-4"> - - - - - - {deprecationBannerTitle(countDistinctModels(deprecated), countDistinctModels(imminent), data.warn_within_days)} - - -
    - {visibleEntries.map((info, index) => ( -
  • {deprecationEntryText(info)}
    • - ))} -
-
- {entries.length > 5 && ( - - )} - - Get Slack alerts for deprecations - -
- - - ); -}; - -export default ModelDeprecationBanner; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.integration.test.tsx deleted file mode 100644 index d2ddb805a0b..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.integration.test.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; -import { afterEach, describe, expect, it, vi } from "vitest"; - -import ModelDeprecationBanner from "./ModelDeprecationBanner"; -import PriceDataManagementTab from "./PriceDataManagementTab"; - -const TEST_TODAY = new Date(2026, 10, 4); - -vi.mock("@/components/price_data_reload", () => ({ - default: ({ onReloadSuccess }: { onReloadSuccess: () => void }) => , -})); -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: () => ({ accessToken: "sk-test", userRole: "Admin" }), -})); - -const renderTabWithBanner = (queryClient: QueryClient) => - render( - - <> - - - - , - ); - -describe("PriceDataManagementTab", () => { - afterEach(() => { - vi.unstubAllGlobals(); - }); - - it("refetches deprecations after reloading price data", async () => { - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - const deprecatedResponse = { - checked_at: "2026-06-01T00:00:00Z", - warn_within_days: 30, - deprecated: [ - { - model_name: "old-model", - litellm_provider: null, - deprecation_date: "2026-06-01", - days_until_deprecation: -1, - status: "deprecated", - successor_model: null, - }, - ], - imminent: [], - upcoming: [], - }; - const currentResponse = { - checked_at: "2026-06-02T00:00:00Z", - warn_within_days: 30, - deprecated: [], - imminent: [ - { - model_name: "current-model", - litellm_provider: null, - deprecation_date: "2026-11-30", - days_until_deprecation: 26, - status: "imminent", - successor_model: null, - }, - ], - upcoming: [], - }; - const fetchMock = vi - .fn() - .mockResolvedValueOnce(Response.json(deprecatedResponse)) - .mockResolvedValueOnce(Response.json(currentResponse)); - vi.stubGlobal("fetch", fetchMock); - renderTabWithBanner(queryClient); - - expect(await screen.findByText("old-model was deprecated on 2026-06-01")).toBeInTheDocument(); - fireEvent.click(screen.getByRole("button", { name: "reload" })); - expect(await screen.findByText("current-model deprecates on 2026-11-30, in 26 days")).toBeInTheDocument(); - await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); - expect( - fetchMock.mock.calls.map( - ([input]) => new URL(input instanceof Request ? input.url : String(input), "http://localhost").pathname, - ), - ).toEqual(["/model/deprecations", "/model/deprecations"]); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx index 445dc505ce6..b2beb5beeed 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx @@ -2,16 +2,13 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { fireEvent, render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; -import { modelDeprecationKeys } from "../../hooks/models/useModelDeprecations"; import { modelCostMapKeys } from "../../hooks/models/useModelCostMap"; import PriceDataManagementTab from "./PriceDataManagementTab"; vi.mock("@/components/price_data_reload", () => ({ default: ({ onReloadSuccess }: { onReloadSuccess: () => void }) => , })); -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: () => ({ accessToken: "sk-test", userRole: "Admin" }), -})); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => ({ accessToken: "sk-test" }) })); const renderTab = (queryClient: QueryClient) => render( @@ -30,16 +27,13 @@ describe("PriceDataManagementTab", () => { const queryClient = new QueryClient(); const liveKey = modelCostMapKeys.list({}); const catalogKey = modelCostMapKeys.list({ filters: { catalog_only: "true" } }); - const deprecationKey = modelDeprecationKeys.all; queryClient.setQueryData(liveKey, {}); queryClient.setQueryData(catalogKey, {}); - queryClient.setQueryData(deprecationKey, {}); renderTab(queryClient); fireEvent.click(screen.getByRole("button", { name: "reload" })); expect(queryClient.getQueryState(liveKey)?.isInvalidated).toBe(true); expect(queryClient.getQueryState(catalogKey)?.isInvalidated).toBe(true); - expect(queryClient.getQueryState(deprecationKey)?.isInvalidated).toBe(true); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.tsx index 4dddc5333ea..1457b5b311f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.tsx @@ -3,7 +3,6 @@ import React from "react"; import { useTranslation } from "react-i18next"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { useQueryClient } from "@tanstack/react-query"; -import { modelDeprecationKeys } from "../../hooks/models/useModelDeprecations"; import { modelCostMapKeys } from "../../hooks/models/useModelCostMap"; const PriceDataManagementTab = () => { @@ -21,10 +20,7 @@ const PriceDataManagementTab = () => { { - void Promise.all([ - queryClient.invalidateQueries({ queryKey: modelCostMapKeys.all }), - queryClient.invalidateQueries({ queryKey: modelDeprecationKeys.all }), - ]); + queryClient.invalidateQueries({ queryKey: modelCostMapKeys.all }); }} buttonText={t("modelsAndEndpoints.priceDataManagementTab.reloadButton")} size="middle" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/deprecationBannerText.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/deprecationBannerText.test.ts deleted file mode 100644 index 66d065e6caa..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/deprecationBannerText.test.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { ModelDeprecationInfo } from "@/app/(dashboard)/hooks/models/useModelDeprecations"; -import { - bucketByLocalDate, - countDistinctModels, - deprecationBannerTitle, - deprecationEntryText, - localDaysUntil, - parseLocalDate, -} from "./deprecationBannerText"; - -const makeInfo = (overrides: Partial = {}): ModelDeprecationInfo => ({ - model_name: "gpt-4o", - deprecation_date: "2026-11-30", - days_until_deprecation: 26, - status: "imminent", - ...overrides, -}); - -describe("local deprecation dates", () => { - it("parses date parts as a local calendar date and counts from local midnight", () => { - const parsed = parseLocalDate("2026-11-06"); - - expect([parsed.getFullYear(), parsed.getMonth(), parsed.getDate(), parsed.getHours()]).toEqual([2026, 10, 6, 0]); - expect(localDaysUntil("2026-11-06", new Date(2026, 10, 4, 18))).toBe(2); - }); - - it("buckets entries by their local date and warning window", () => { - const buckets = bucketByLocalDate( - [ - makeInfo({ model_name: "yesterday", deprecation_date: "2026-11-03", status: "imminent" }), - makeInfo({ model_name: "today", deprecation_date: "2026-11-04", status: "deprecated" }), - makeInfo({ model_name: "outside-window", deprecation_date: "2026-12-05", status: "imminent" }), - ], - new Date(2026, 10, 4), - 30, - ); - - expect( - buckets.deprecated.map(({ model_name, days_until_deprecation, status }) => [ - model_name, - days_until_deprecation, - status, - ]), - ).toEqual([["yesterday", -1, "deprecated"]]); - expect( - buckets.imminent.map(({ model_name, days_until_deprecation, status }) => [ - model_name, - days_until_deprecation, - status, - ]), - ).toEqual([["today", 0, "imminent"]]); - }); -}); - -describe("countDistinctModels", () => { - it("counts visible names and providers, not deployment aliases", () => { - expect( - countDistinctModels([ - makeInfo({ model_name: "team-a", team_public_model_name: "shared", litellm_provider: "openai" }), - makeInfo({ model_name: "team-b", team_public_model_name: "shared", litellm_provider: "openai" }), - makeInfo({ model_name: "team-c", team_public_model_name: "shared", litellm_provider: "anthropic" }), - ]), - ).toBe(2); - }); -}); - -describe("deprecationBannerTitle", () => { - it("formats both counts with singular and plural wording", () => { - expect(deprecationBannerTitle(2, 1, 30)).toBe("2 models are deprecated, 1 deprecates within 30 days"); - expect(deprecationBannerTitle(1, 2, 30)).toBe("1 model is deprecated, 2 deprecate within 30 days"); - }); - - it("omits empty counts and uses singular model wording", () => { - expect(deprecationBannerTitle(1, 0, 30)).toBe("1 model is deprecated"); - expect(deprecationBannerTitle(0, 1, 30)).toBe("1 model deprecates within 30 days"); - expect(deprecationBannerTitle(0, 3, 30)).toBe("3 models deprecate within 30 days"); - expect(deprecationBannerTitle(0, 0, 30)).toBe(""); - }); -}); - -describe("deprecationEntryText", () => { - it("formats deprecated entries with provider and successor", () => { - const deprecatedInfo: ModelDeprecationInfo = { - model_name: "gpt-4", - litellm_provider: "openai", - deprecation_date: "2026-06-01", - days_until_deprecation: -152, - status: "deprecated", - successor_model: "gpt-5.6-sol", - }; - - expect(deprecationEntryText(deprecatedInfo)).toBe( - "gpt-4 (openai) was deprecated on 2026-06-01, migrate to gpt-5.6-sol", - ); - }); - - it("omits a null provider and successor", () => { - expect( - deprecationEntryText( - makeInfo({ - litellm_provider: null, - successor_model: null, - }), - ), - ).toBe("gpt-4o deprecates on 2026-11-30, in 26 days"); - }); - - it("uses singular wording for one day", () => { - expect( - deprecationEntryText( - makeInfo({ - days_until_deprecation: 1, - }), - ), - ).toBe("gpt-4o deprecates on 2026-11-30, in 1 day"); - }); - - it("says an imminent model deprecates today", () => { - expect( - deprecationEntryText( - makeInfo({ - days_until_deprecation: 0, - deprecation_date: "2026-11-04", - }), - ), - ).toBe("gpt-4o deprecates today (2026-11-04)"); - }); - - it("uses the team-visible model name when present", () => { - expect( - deprecationEntryText( - makeInfo({ - model_name: "internal-name", - team_public_model_name: "team-name", - }), - ), - ).toBe("team-name deprecates on 2026-11-30, in 26 days"); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/deprecationBannerText.ts b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/deprecationBannerText.ts deleted file mode 100644 index 0c035c5bdd6..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/deprecationBannerText.ts +++ /dev/null @@ -1,96 +0,0 @@ -import type { ModelDeprecationInfo } from "@/app/(dashboard)/hooks/models/useModelDeprecations"; - -const shownModelName = (info: ModelDeprecationInfo): string => info.team_public_model_name ?? info.model_name; - -export const countDistinctModels = (infos: readonly ModelDeprecationInfo[]): number => - new Set(infos.map((info) => JSON.stringify([shownModelName(info), info.litellm_provider ?? null]))).size; - -export const parseLocalDate = (value: string): Date => { - const [year, month, day] = value.split("-"); - return new Date(Number(year), Number(month) - 1, Number(day)); -}; - -const startOfLocalDay = (today: Date): Date => new Date(today.getFullYear(), today.getMonth(), today.getDate()); - -export const localDaysUntil = (deprecationDate: string, today: Date): number => - Math.round((parseLocalDate(deprecationDate).getTime() - startOfLocalDay(today).getTime()) / 86_400_000); - -export const bucketByLocalDate = ( - infos: readonly ModelDeprecationInfo[], - today: Date, - warnWithinDays: number, -): { deprecated: ModelDeprecationInfo[]; imminent: ModelDeprecationInfo[] } => { - const inWindow = infos - .map((info): ModelDeprecationInfo => { - const daysUntilDeprecation = localDaysUntil(info.deprecation_date, today); - return { - ...info, - days_until_deprecation: daysUntilDeprecation, - status: daysUntilDeprecation < 0 ? "deprecated" : "imminent", - }; - }) - .filter((info) => info.days_until_deprecation <= warnWithinDays); - - return { - deprecated: inWindow.filter((info) => info.status === "deprecated"), - imminent: inWindow.filter((info) => info.status === "imminent"), - }; -}; - -const hasSameDedupeKey = (candidate: ModelDeprecationInfo, info: ModelDeprecationInfo): boolean => { - const sameDisplayedModel = shownModelName(candidate) === shownModelName(info); - const sameProvider = (candidate.litellm_provider ?? null) === (info.litellm_provider ?? null); - const sameDeprecationDate = candidate.deprecation_date === info.deprecation_date; - const sameSuccessor = (candidate.successor_model ?? null) === (info.successor_model ?? null); - const sameVisibleIdentity = sameDisplayedModel && sameProvider; - const sameDeprecationIdentity = sameDeprecationDate && sameSuccessor; - return sameVisibleIdentity && sameDeprecationIdentity; -}; - -export const dedupeDeprecationEntries = (infos: readonly ModelDeprecationInfo[]): ModelDeprecationInfo[] => - infos.filter((info, index) => infos.findIndex((candidate) => hasSameDedupeKey(candidate, info)) === index); - -export const deprecationBannerFingerprint = (infos: readonly ModelDeprecationInfo[]): string => - JSON.stringify( - infos - .map((info) => - JSON.stringify([ - shownModelName(info), - info.litellm_provider ?? null, - info.deprecation_date, - info.status, - info.successor_model ?? null, - ]), - ) - .sort(), - ); - -export const deprecationBannerTitle = ( - deprecatedCount: number, - imminentCount: number, - warnWithinDays: number, -): string => { - const deprecatedText = - deprecatedCount === 0 ? "" : `${deprecatedCount} ${deprecatedCount === 1 ? "model is" : "models are"} deprecated`; - const imminentModelLabel = imminentCount === 1 ? "model" : "models"; - const imminentSubject = deprecatedCount > 0 ? `${imminentCount}` : `${imminentCount} ${imminentModelLabel}`; - const imminentVerb = imminentCount === 1 ? "deprecates" : "deprecate"; - const imminentText = imminentCount === 0 ? "" : `${imminentSubject} ${imminentVerb} within ${warnWithinDays} days`; - return [deprecatedText, imminentText].filter((text) => text.length > 0).join(", "); -}; - -export const deprecationEntryText = (info: ModelDeprecationInfo): string => { - const model = `${shownModelName(info)}${info.litellm_provider ? ` (${info.litellm_provider})` : ""}`; - const successor = info.successor_model ? `, migrate to ${info.successor_model}` : ""; - - if (info.status === "deprecated") { - return `${model} was deprecated on ${info.deprecation_date}${successor}`; - } - - if (info.days_until_deprecation === 0) { - return `${model} deprecates today (${info.deprecation_date})${successor}`; - } - - const dayLabel = info.days_until_deprecation === 1 ? "day" : "days"; - return `${model} deprecates on ${info.deprecation_date}, in ${info.days_until_deprecation} ${dayLabel}${successor}`; -}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.integration.test.tsx index 7aeff6f44f2..4d18966625e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.integration.test.tsx @@ -1,9 +1,9 @@ /* @vitest-environment jsdom */ -import { screen, waitForElementToBeRemoved } from "@testing-library/react"; +import { screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import * as networking from "@/components/networking"; -import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils"; +import { renderWithProviders } from "../../../../tests/test-utils"; import ModelsAndEndpointsPage from "./page"; vi.mock("./panels/AllModelsPanel", () => ({ default: () =>
})); @@ -201,7 +201,6 @@ const createMockTeamData = (overrides = {}) => ({ describe("ModelsAndEndpointsPage ?team drill-in", () => { beforeEach(() => { - testQueryClient.clear(); authState.userRole = "Admin"; authState.isViewOnly = false; can.mockReturnValue(true); @@ -222,7 +221,6 @@ describe("ModelsAndEndpointsPage ?team drill-in", () => { }); afterEach(() => { - vi.unstubAllGlobals(); vi.clearAllMocks(); }); @@ -246,53 +244,4 @@ describe("ModelsAndEndpointsPage ?team drill-in", () => { expect(screen.queryByRole("button", { name: /edit settings/i })).not.toBeInTheDocument(); expect(screen.queryByText("Disable all global guardrails")).not.toBeInTheDocument(); }); - - it("refreshes deprecation banner data when the models list is refreshed", async () => { - const deprecatedResponse = { - checked_at: "2026-06-01T00:00:00Z", - warn_within_days: 30, - deprecated: [ - { - model_name: "gpt-4", - litellm_provider: "openai", - deprecation_date: "2026-06-01", - days_until_deprecation: -1, - status: "deprecated", - successor_model: "gpt-5", - }, - ], - imminent: [], - upcoming: [], - }; - const currentResponse = { - checked_at: "2026-06-02T00:00:00Z", - warn_within_days: 30, - deprecated: [], - imminent: [], - upcoming: [], - }; - const fetchMock = vi - .fn() - .mockImplementationOnce(async (input) => { - const requestUrl = input instanceof Request ? input.url : String(input); - expect(new URL(requestUrl, "http://localhost").pathname).toBe("/model/deprecations"); - return Response.json(deprecatedResponse); - }) - .mockImplementationOnce(async (input) => { - const requestUrl = input instanceof Request ? input.url : String(input); - expect(new URL(requestUrl, "http://localhost").pathname).toBe("/model/deprecations"); - return Response.json(currentResponse); - }); - vi.stubGlobal("fetch", fetchMock); - - const user = userEvent.setup({ delay: null }); - renderWithProviders(); - - expect( - await screen.findByText("gpt-4 (openai) was deprecated on 2026-06-01, migrate to gpt-5"), - ).toBeInTheDocument(); - await user.click(screen.getByRole("button", { name: "Refresh models" })); - await waitForElementToBeRemoved(() => screen.queryByRole("alert")); - expect(fetchMock).toHaveBeenCalledTimes(2); - }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx index be8ee1bb8c4..3cfed7297c8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx @@ -29,9 +29,6 @@ vi.mock("./detailNavigation", () => ({ vi.mock("@/components/molecules/DecisionModelsBanner", () => ({ default: () =>
, })); -vi.mock("./components/ModelDeprecationBanner", () => ({ - default: () =>
, -})); vi.mock("@/components/model_info_view", () => ({ default: ({ modelId }: { modelId: string }) =>
model:{modelId}
, })); @@ -112,12 +109,6 @@ describe("ModelsAndEndpointsPage", () => { expect(screen.queryByRole("region", { name: "Decision models" })).not.toBeInTheDocument(); }); - it("renders the model deprecation banner", () => { - renderPage(); - - expect(screen.getByRole("region", { name: "Model deprecations" })).toBeInTheDocument(); - }); - it("renders the model detail overlay from the ?model drill-in and hides the tabs", () => { detailState.modelId = "abc-123"; renderPage(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx index b4082a14ffe..e0e5db8a8fd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.tsx @@ -7,7 +7,6 @@ import { useQueryClient } from "@tanstack/react-query"; import type { ParseKeys } from "i18next"; import { useTranslation } from "react-i18next"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { invalidateModelQueries } from "@/app/(dashboard)/hooks/models/invalidateModelQueries"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings"; import { useUrlTab } from "@/hooks/useUrlTab"; @@ -31,7 +30,6 @@ import AccessGroupBudgetsPanel from "@/app/(dashboard)/models-and-endpoints/pane import PriceDataPanel from "@/app/(dashboard)/models-and-endpoints/panels/PriceDataPanel"; import { Button } from "@/components/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import ModelDeprecationBanner from "./components/ModelDeprecationBanner"; const MODEL_TAB_SLUGS = [ "add", @@ -168,10 +166,10 @@ export default function ModelsAndEndpointsPage() { const handleRefreshClick = () => { setLastRefreshed(new Date().toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" })); - void invalidateModelQueries(queryClient); + queryClient.invalidateQueries({ queryKey: ["models", "list"] }); }; - const invalidateModels = () => invalidateModelQueries(queryClient); + const invalidateModels = () => queryClient.invalidateQueries({ queryKey: ["models", "list"] }); if (teamId) { return ( @@ -206,7 +204,6 @@ export default function ModelsAndEndpointsPage() {
{!isViewOnly && } - {modelId ? ( { +const setup = async (searchParams?: string, onUrlUpdate?: OnUrlUpdateFunction) => { const user = userEvent.setup({ pointerEventsCheck: PointerEventsCheckLevel.Never }); - renderWithProviders( - showBanner ? ( - <> - - - - ) : ( - - ), - { searchParams, onUrlUpdate }, - ); + renderWithProviders(, { searchParams, onUrlUpdate }); await screen.findByText("Provider"); const openAdvanced = async () => { @@ -200,69 +186,11 @@ const setup = async (searchParams?: string, onUrlUpdate?: OnUrlUpdateFunction, s describe("AddModelPanel submit payload contract", () => { beforeEach(() => { vi.clearAllMocks(); - testQueryClient.clear(); mockUseModelCostMap.mockReturnValue({ data: {} }); mockPtuEnabled.mockReturnValue(false); mockAuthorized.mockReturnValue(PROXY_ADMIN); }); - afterEach(() => { - vi.unstubAllGlobals(); - }); - - it("refreshes deprecation banner data after adding a model", async () => { - const deprecatedResponse = { - checked_at: "2026-06-01T00:00:00Z", - warn_within_days: 30, - deprecated: [ - { - model_name: "old-model", - litellm_provider: null, - deprecation_date: "2026-06-01", - days_until_deprecation: -1, - status: "deprecated", - successor_model: null, - }, - ], - imminent: [], - upcoming: [], - }; - const currentResponse = { - checked_at: "2026-06-02T00:00:00Z", - warn_within_days: 30, - deprecated: [], - imminent: [ - { - model_name: "current-model", - litellm_provider: null, - deprecation_date: "2026-11-30", - days_until_deprecation: 26, - status: "imminent", - successor_model: null, - }, - ], - upcoming: [], - }; - const fetchMock = vi - .fn() - .mockResolvedValueOnce(Response.json(deprecatedResponse)) - .mockResolvedValueOnce(Response.json(currentResponse)); - vi.stubGlobal("fetch", fetchMock); - - const { fillRequired, submit } = await setup(undefined, undefined, true); - - expect(await screen.findByText("old-model was deprecated on 2026-06-01")).toBeInTheDocument(); - await fillRequired(); - await submit(); - expect(await screen.findByText("current-model deprecates on 2026-11-30, in 26 days")).toBeInTheDocument(); - expect(fetchMock).toHaveBeenCalledTimes(2); - expect( - fetchMock.mock.calls.map( - ([input]) => new URL(input instanceof Request ? input.url : String(input), "http://localhost").pathname, - ), - ).toEqual(["/model/deprecations", "/model/deprecations"]); - }); - it("lists catalog models in the model picker, not entries registered at runtime for deployments", async () => { mockUseModelCostMap.mockImplementation((_enabled: boolean, catalogOnly: boolean) => ({ data: catalogOnly diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx index 13bec9506e4..392a3c3940a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx @@ -15,7 +15,6 @@ import { Providers, getPlaceholder, getProviderModels } from "@/components/provi import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import { useCredentials } from "@/app/(dashboard)/hooks/credentials/useCredentials"; import { useProviderFields } from "@/app/(dashboard)/hooks/providers/useProviderFields"; -import { invalidateModelQueries } from "@/app/(dashboard)/hooks/models/invalidateModelQueries"; import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { buildDecisionCatalog } from "@/lib/decisionModels"; @@ -66,7 +65,7 @@ export default function AddModelPanel() { const decisionCatalog = useMemo(() => buildDecisionCatalog(modelCostMapData), [modelCostMapData]); const [showAdvancedSettings, setShowAdvancedSettings] = useState(false); - const refresh = () => invalidateModelQueries(queryClient); + const refresh = () => queryClient.invalidateQueries({ queryKey: ["models", "list"] }); const mountedValues = () => projectMountedValues(registry, form.getValues); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.test.tsx index f4ff11494da..2e27a258a84 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.test.tsx @@ -1,9 +1,7 @@ import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; -import { QueryClient, QueryClientProvider, useQuery } from "@tanstack/react-query"; import userEvent from "@testing-library/user-event"; -import { useState, type ReactNode } from "react"; +import { useState } from "react"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { modelDeprecationKeys } from "@/app/(dashboard)/hooks/models/useModelDeprecations"; import AgentBuilderView from "./AgentBuilderView"; import type { AgentModel } from "../../llm_calls/fetch_agents"; @@ -69,41 +67,10 @@ const props = { userRole: "Admin", }; -const modelListKey = ["models", "list"] as const; - -const ModelQueryProbe = ({ - modelListQuery, - modelDeprecationQuery, -}: { - modelListQuery: () => Promise; - modelDeprecationQuery: () => Promise; -}) => { - useQuery({ queryKey: modelListKey, queryFn: modelListQuery, staleTime: Infinity }); - useQuery({ queryKey: modelDeprecationKeys.all, queryFn: modelDeprecationQuery, staleTime: Infinity }); - return null; -}; - const controlUnder = (label: string): HTMLElement => within(screen.getByText(label).parentElement!).getByRole("combobox"); -const renderWithQueryClient = (children: ReactNode, queryClient = new QueryClient()) => { - return render({children}); -}; - -const renderView = (queryClient?: QueryClient) => renderWithQueryClient(, queryClient); - -const renderViewWithModelQueryProbe = ( - queryClient: QueryClient, - modelListQuery: () => Promise, - modelDeprecationQuery: () => Promise, -) => - renderWithQueryClient( - <> - - - , - queryClient, - ); +const renderView = () => render(); const waitForRoster = async () => { await screen.findByRole("button", { name: "support-agent litellm_agent" }); @@ -132,7 +99,7 @@ beforeEach(() => { describe("AgentBuilderView", () => { it("asks the visitor to sign in when there is no session", () => { - renderWithQueryClient(); + render(); expect(screen.getByText("Sign in to use Agent Builder.")).toBeInTheDocument(); expect(fetchAvailableAgentModels).not.toHaveBeenCalled(); @@ -203,25 +170,6 @@ describe("AgentBuilderView", () => { expect(payload.litellm_params.model).toBe("litellm_agent/gpt-4o"); }); - it("refetches model queries after creating an agent", async () => { - const user = userEvent.setup(); - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - const modelListQuery = vi.fn().mockResolvedValue([]); - const modelDeprecationQuery = vi.fn().mockResolvedValue({}); - renderViewWithModelQueryProbe(queryClient, modelListQuery, modelDeprecationQuery); - await waitForRoster(); - await waitFor(() => expect(modelListQuery).toHaveBeenCalledTimes(1)); - await waitFor(() => expect(modelDeprecationQuery).toHaveBeenCalledTimes(1)); - - await user.click(screen.getByRole("button", { name: /New agent/i })); - fireEvent.change(screen.getByPlaceholderText("My Agent"), { target: { value: "billing-agent" } }); - await user.click(screen.getByRole("button", { name: /Save Agent/i })); - - await waitFor(() => expect(modelCreateCall).toHaveBeenCalled()); - await waitFor(() => expect(modelListQuery).toHaveBeenCalledTimes(2)); - await waitFor(() => expect(modelDeprecationQuery).toHaveBeenCalledTimes(2)); - }); - it("will not save a draft without a name", async () => { const user = userEvent.setup(); renderView(); @@ -245,24 +193,6 @@ describe("AgentBuilderView", () => { expect(modelPatchUpdateCall.mock.calls[0][2]).toBe("agent-1"); }); - it("refetches model queries after updating an agent", async () => { - const user = userEvent.setup(); - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - const modelListQuery = vi.fn().mockResolvedValue([]); - const modelDeprecationQuery = vi.fn().mockResolvedValue({}); - renderViewWithModelQueryProbe(queryClient, modelListQuery, modelDeprecationQuery); - await waitForRoster(); - await screen.findByDisplayValue("support-agent"); - await waitFor(() => expect(modelListQuery).toHaveBeenCalledTimes(1)); - await waitFor(() => expect(modelDeprecationQuery).toHaveBeenCalledTimes(1)); - - await user.click(screen.getByRole("button", { name: /Update Agent/i })); - - await waitFor(() => expect(modelPatchUpdateCall).toHaveBeenCalled()); - await waitFor(() => expect(modelListQuery).toHaveBeenCalledTimes(2)); - await waitFor(() => expect(modelDeprecationQuery).toHaveBeenCalledTimes(2)); - }); - it("deletes only after the warning is confirmed", async () => { const user = userEvent.setup(); renderView(); @@ -280,27 +210,6 @@ describe("AgentBuilderView", () => { await waitFor(() => expect(modelDeleteCall).toHaveBeenCalledWith("sk-access", "agent-1")); }); - it("refetches model queries after deleting an agent", async () => { - const user = userEvent.setup(); - const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }); - const modelListQuery = vi.fn().mockResolvedValue([]); - const modelDeprecationQuery = vi.fn().mockResolvedValue({}); - renderViewWithModelQueryProbe(queryClient, modelListQuery, modelDeprecationQuery); - await waitForRoster(); - await screen.findByDisplayValue("support-agent"); - await waitFor(() => expect(modelListQuery).toHaveBeenCalledTimes(1)); - await waitFor(() => expect(modelDeprecationQuery).toHaveBeenCalledTimes(1)); - - await user.click(screen.getAllByRole("button", { name: /Delete$/ })[0]); - await screen.findByText(/Are you sure you want to delete "support-agent"/); - const deleteButtons = screen.getAllByRole("button", { name: /Delete$/ }); - await user.click(deleteButtons[deleteButtons.length - 1]); - - await waitFor(() => expect(modelDeleteCall).toHaveBeenCalledWith("sk-access", "agent-1")); - await waitFor(() => expect(modelListQuery).toHaveBeenCalledTimes(2)); - await waitFor(() => expect(modelDeprecationQuery).toHaveBeenCalledTimes(2)); - }); - it("abandons the delete when the warning is dismissed", async () => { const user = userEvent.setup(); renderView(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.tsx index c79c539cac9..f9909462ee9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/AgentBuilderView.tsx @@ -3,8 +3,6 @@ import { Bot, FlaskConical, Link as LinkIcon, MessageSquare, Plus, Save, Trash2 } from "lucide-react"; import React, { useCallback, useEffect, useState } from "react"; import { Trans, useTranslation } from "react-i18next"; -import { useQueryClient } from "@tanstack/react-query"; -import { invalidateModelQueries } from "@/app/(dashboard)/hooks/models/invalidateModelQueries"; import { AlertDialog, AlertDialogAction, @@ -205,7 +203,6 @@ export default function AgentBuilderView({ customProxyBaseUrl, }: AgentBuilderViewProps) { const { t } = useTranslation(); - const queryClient = useQueryClient(); const [agentModels, setAgentModels] = useState([]); const [modelGroups, setModelGroups] = useState([]); const [loadingAgents, setLoadingAgents] = useState(true); @@ -361,7 +358,6 @@ export default function AgentBuilderView({ }, model_info: {}, }); - void invalidateModelQueries(queryClient); // /model/new returns the row with `model_id` at the top level. // Prefer that id over name-matching so we land on the just-created // agent even when its public name collides with another team's. @@ -401,7 +397,6 @@ export default function AgentBuilderView({ }, selectedAgentModelId, ); - void invalidateModelQueries(queryClient); toast.success(t("playground.chatUi.agentBuilderView.updated")); const list = await loadAgents(); const stillSelected = list.find((a) => getAgentModelId(a) === selectedAgentModelId); @@ -447,7 +442,6 @@ export default function AgentBuilderView({ setDeleting(true); try { await modelDeleteCall(accessToken, selectedAgentModelId); - void invalidateModelQueries(queryClient); toast.success(t("playground.chatUi.agentBuilderView.deleted")); const list = await loadAgents(); const remaining = list.filter((a) => getAgentModelId(a) !== selectedAgentModelId); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts index 1dfdfd1da78..24a0ba0d86a 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts @@ -210,3 +210,27 @@ describe("model colors", () => { expect(bucketTotals(series).get("2026-10-01")).toBe(55); }); }); + +describe("daily key stacks", () => { + it("should preserve each day's total when truncated keys are grouped into Other", () => { + const first = day("2026-11-01", 10, {}); + first.metrics.api_requests = 100; + first.breakdown.api_keys = { + "key-a": { metrics: { ...metrics(3), api_requests: 30 }, metadata: { key_alias: "Support", team_id: null } }, + "key-b": { metrics: { ...metrics(2), api_requests: 20 }, metadata: { key_alias: "Research", team_id: null } }, + }; + const second = day("2026-11-02", 15, {}); + second.metrics.api_requests = 150; + second.breakdown.api_keys = { + "key-a": { metrics: { ...metrics(1), api_requests: 10 }, metadata: { key_alias: "Support", team_id: null } }, + "key-b": { metrics: { ...metrics(8), api_requests: 80 }, metadata: { key_alias: "Research", team_id: null } }, + }; + const result = seriesBy([first, second], "api_keys", "requests", 1); + expect(result.labels).toEqual(["key-b", "Other"]); + expect(result.data.map((row) => [row[result.keys[0]], row[result.keys[1]]])).toEqual([ + [20, 80], + [80, 70], + ]); + expect([...bucketTotals(result).values()]).toEqual([100, 150]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts index 54f4e5dd1de..2943302dfae 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts @@ -21,7 +21,10 @@ interface RankedRow extends Record { const EMPTY_RANKED = { spend: 0, requests: 0, successful_requests: 0, failed_requests: 0, tokens: 0 } as const; /** Sums one breakdown dimension across every day, highest spend first. */ -export const rollUpBreakdown = (results: readonly DailyData[], dimension: BreakdownDimension): RankedRow[] => { +export const rollUpBreakdown = ( + results: readonly DailyData[], + dimension: BreakdownDimension | "api_keys", +): RankedRow[] => { const entries = results.flatMap((day) => Object.entries(day.breakdown[dimension] ?? {})); const totals = entries.reduce>((acc, [key, entry]) => { const row = acc.get(key) ?? { key, ...EMPTY_RANKED }; @@ -117,7 +120,7 @@ export const rankValue = (row: RankedRow, metric: UsageMetric): number => { */ export const seriesBy = ( results: readonly DailyData[], - dimension: BreakdownDimension, + dimension: BreakdownDimension | "api_keys", metric: UsageMetric, options: number | { top?: number; colorByKey?: ReadonlyMap } = 6, ): Series => { diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx new file mode 100644 index 00000000000..2b3be5904e4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx @@ -0,0 +1,108 @@ +"use client"; + +import { useMemo, useState } from "react"; +import type { ColumnDef } from "@tanstack/react-table"; +import { DataTable, DataTableToolbar } from "@/components/shared/DataTable"; +import { PageTabs, PageTabsList, PageTabsTrigger, PageTabsContent } from "@/components/shared/Page"; +import type { DeprecationAlert } from "./deprecationAlerts"; +import { AlertStatus, effectiveAlertStatus } from "./migrationState"; +import RecommendedReplacement from "./RecommendedReplacement"; +import { DeprecationModel, DeprecationTiming } from "./DeprecationModel"; +import { migrationHref } from "./migrationRoute"; + +const columns: ColumnDef[] = [ + { + id: "model", + header: "Model", + cell: ({ row }) => ( +
+ + +
+ ), + }, + { + id: "deprecation", + header: "Deprecation", + cell: ({ row }) => ( +
+ + {row.original.model.deprecation_date} +
+ ), + }, + { accessorKey: "requests", header: "Requests · 30d", cell: ({ row }) => row.original.requests.toLocaleString() }, + { accessorKey: "lastUsedDate", header: "Last seen (UTC)" }, + { + id: "review", + header: "", + cell: ({ row }) => ( + + Review impact → + + ), + }, +]; + +export default function AlertsInbox({ + alerts, + pending, + status, + onStatusChange, + onRefresh, + refreshing, +}: { + alerts: readonly DeprecationAlert[]; + pending: boolean; + status: AlertStatus; + onStatusChange: (status: AlertStatus) => void; + onRefresh: () => void; + refreshing: boolean; +}) { + const [search, setSearch] = useState(""); + const counts = { open: 0, resolved: 0, dismissed: 0 }; + for (const alert of alerts) counts[effectiveAlertStatus(alert)]++; + const filtered = useMemo( + () => + alerts.filter( + (alert) => + effectiveAlertStatus(alert) === status && alert.usageModel.toLowerCase().includes(search.toLowerCase()), + ), + [alerts, status, search], + ); + return ( + + + {Object.values(AlertStatus).map((value) => ( + + {value} + {counts[value]} + + ))} + + + row.id} + fillHeight + size="compact" + isLoading={pending} + loadingMessage="Loading model deprecations…" + noDataMessage={`No ${status} model deprecations in the last 30 days.`} + toolbar={(table) => ( + + )} + /> + + + ); +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx new file mode 100644 index 00000000000..dc2d2a848be --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx @@ -0,0 +1,35 @@ +"use client"; + +import { ArrowUpRight } from "lucide-react"; +import { DeprecationModel, DeprecationTiming } from "./DeprecationModel"; +import { buttonVariants } from "@/components/ui/button"; +import { cn } from "@/lib/cva.config"; +import { migrationHref } from "@/components/ModelDeprecationAlerts/migrationRoute"; +import RecommendedReplacement from "./RecommendedReplacement"; +import type { DeprecationAlert } from "./deprecationAlerts"; + +export default function DeprecationAlertItem({ alert }: { alert: DeprecationAlert }) { + const retired = alert.model.status === "deprecated"; + const title = retired + ? `${alert.usageModel} is deprecated` + : `${alert.usageModel} retires in ${alert.model.days_until_deprecation} days`; + return ( +
+
+
+ +

+ +

+

+ {alert.requests.toLocaleString()} requests in the last 30 days · Last seen {alert.lastUsedDate} UTC +

+ +
+
+ + Review migration + +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationModel.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationModel.tsx new file mode 100644 index 00000000000..dfb0e2b71c3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationModel.tsx @@ -0,0 +1,32 @@ +import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; +import type { DeprecationAlert } from "./deprecationAlerts"; +import { migrationHref } from "./migrationRoute"; + +export function deprecationProvider(alert: DeprecationAlert): string { + return alert.model.litellm_provider || alert.usageModel.split("/")[0]; +} + +export function DeprecationModel({ alert }: { alert: DeprecationAlert }) { + const provider = deprecationProvider(alert); + return ( + + + {alert.usageModel} + + ); +} + +export function DeprecationTiming({ alert }: { alert: DeprecationAlert }) { + const days = alert.model.days_until_deprecation; + let label = `Deprecates in ${days} ${days === 1 ? "day" : "days"}`; + if (days === 0) label = "Deprecates today"; + if (days < 0) label = `Deprecated ${Math.abs(days)} ${days === -1 ? "day" : "days"} ago`; + return ( + + {label} + + ); +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/MigrationDetail.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/MigrationDetail.tsx new file mode 100644 index 00000000000..0ad842b9078 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/MigrationDetail.tsx @@ -0,0 +1,335 @@ +"use client"; + +import { useMemo } from "react"; +import { useQueryStates } from "nuqs"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import type { ColumnDef } from "@tanstack/react-table"; +import { ArrowUpRight, Check, Copy, RotateCcw, X } from "lucide-react"; +import { useTeams } from "@/app/(dashboard)/hooks/teams/useTeams"; +import { DataTable, useUrlTableState } from "@/components/shared/DataTable"; +import { Button, buttonVariants } from "@/components/ui/button"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { backgroundFetchClient } from "@/lib/http/api"; +import type { components } from "@/lib/http/schema"; +import { keyDetailHref, modelGroupHref } from "@/utils/entityLinks"; +import { copyToClipboard } from "@/utils/dataUtils"; +import { AlertStatus, effectiveAlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; +import { migrationUrlParsers } from "./migrationRoute"; +import type { DeprecationAlert, deprecationUsageRange } from "@/components/ModelDeprecationAlerts/deprecationAlerts"; +import RecommendedReplacement from "@/components/ModelDeprecationAlerts/RecommendedReplacement"; +import { DeprecationTiming } from "./DeprecationModel"; +import UsageTags from "./UsageTags"; +import ModelUsageChart from "./ModelUsageChart"; +import { affectedTeams } from "./migrationData"; + +const TABLE_STATE = { sortFields: [], defaultSort: { id: "", desc: false }, defaultPageSize: 50, filterColumns: [] }; +type KeyRow = components["schemas"]["KeySpendActivityRow"]; + +function migrationHeading(alert: DeprecationAlert, status: AlertStatus): string { + if (status === AlertStatus.Resolved) return "Migration marked resolved"; + if (status === AlertStatus.Dismissed) return "Model deprecation dismissed"; + if (alert.model.status === "deprecated") return "Migration needed: model is deprecated"; + return `Migration due ${alert.model.deprecation_date}`; +} + +function migrationFollowUp( + alert: DeprecationAlert, + context: { team: string; count: number | undefined; lastSeen: string; url: string }, +): string { + const retirement = alert.model.status === "deprecated" ? "retired on" : "retires on"; + const keys = context.count === undefined ? "the affected" : context.count; + return `${context.team}: please migrate ${keys} virtual keys using ${alert.usageModel}. This model ${retirement} ${alert.model.deprecation_date}. Last seen ${context.lastSeen} UTC. Please confirm once migration is complete. Affected keys: ${context.url}`; +} + +export default function MigrationDetail({ + selected, + scope, + range, + isViewOnly, +}: { + selected: DeprecationAlert; + scope: string; + range: ReturnType; + isViewOnly: boolean; +}) { + const [params, setParams] = useQueryStates(migrationUrlParsers); + const { data: teams = [] } = useTeams(); + const teamNames = useMemo( + () => new Map(teams.map((team) => [team.team_id, team.team_alias || team.team_id])), + [teams], + ); + const queryClient = useQueryClient(); + const table = useUrlTableState(TABLE_STATE); + const model = selected.usageModel; + const keyQuery = useQuery({ + queryKey: ["deprecationImpactKeys", scope, model, params.team, range, table.pagination], + + queryFn: async ({ signal }) => { + const query = { + ...range, + model, + offset: table.pagination.pageIndex * table.pagination.pageSize, + limit: table.pagination.pageSize, + timezone: 0, + }; + if (params.team) { + const { data } = await backgroundFetchClient.GET("/team/daily/activity/aggregated/keys", { + params: { query: { ...query, team_ids: params.team } }, + signal, + }); + if (!data) throw new Error("Affected keys are unavailable"); + return data; + } + const { data } = await backgroundFetchClient.GET("/user/daily/activity/aggregated/keys", { + params: { query }, + signal, + }); + if (!data) throw new Error("Affected keys are unavailable"); + return data; + }, + }); + const keys = useMemo(() => keyQuery.data?.api_keys.map((key) => key.api_key) ?? [], [keyQuery.data]); + const tagQueryOptions = { + queryKey: ["deprecationUsageTags", scope, model, range, keys], + enabled: keys.length > 0, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const { data } = await backgroundFetchClient.GET("/alerting/model-deprecations/tags", { + params: { query: { ...range, model, api_keys: keys } }, + signal, + }); + if (!data) throw new Error("Usage tags are unavailable"); + return data; + }, + staleTime: 5 * 60 * 1000, + }; + const tagQuery = useQuery(tagQueryOptions); + const tagsByKey = useMemo(() => { + const tags = new Map(); + for (const usage of tagQuery.data ?? []) { + const existing = tags.get(usage.api_key) ?? []; + existing.push(usage.tag); + tags.set(usage.api_key, existing); + } + return tags; + }, [tagQuery.data]); + const teamQuery = useQuery({ + queryKey: ["deprecationImpactTeams", scope, model, range], + + queryFn: async ({ signal }) => { + const query = { ...range, model, timezone: 0, api_key_limit: 1 }; + const { data } = await backgroundFetchClient.GET("/team/daily/activity/aggregated", { + params: { query }, + signal, + }); + if (!data) throw new Error("Affected teams are unavailable"); + return affectedTeams(data); + }, + }); + const statusMutation = useMutation({ + mutationFn: async (status: AlertStatus) => { + const body = { + model: selected.usageModel, + deprecation_date: selected.model.deprecation_date, + status, + last_seen_date: selected.lastUsedDate, + last_seen_requests: selected.lastDayRequests, + }; + const { data } = await backgroundFetchClient.PUT("/alerting/model-deprecations/state", { body }); + if (!data) throw new Error("Could not save alert status"); + return data; + }, + onSuccess: () => queryClient.invalidateQueries({ queryKey: ["deprecationAlerts"] }), + }); + const columns = useMemo[]>( + () => [ + { + id: "key", + header: "Virtual key", + cell: ({ row }) => ( + + {row.original.metadata.key_alias || "Unnamed key"} + + ), + }, + { + id: "team", + header: "Team", + cell: ({ row }) => + teamNames.get(row.original.metadata.team_id ?? "") || row.original.metadata.team_id || "No team", + }, + { + id: "owner", + header: "Owner", + cell: ({ row }) => row.original.metadata.user_email || row.original.metadata.user_id || "Not recorded", + }, + { + id: "tags", + header: "Usage tags", + cell: ({ row }) => ( + + ), + }, + { + id: "requests", + header: "Requests · 30d", + cell: ({ row }) => {row.original.metrics.api_requests.toLocaleString()}, + }, + { id: "spend", header: "Spend · 30d", cell: ({ row }) => `$${row.original.metrics.spend.toFixed(2)}` }, + ], + [teamNames, tagsByKey, tagQuery.isPending, tagQuery.isError], + ); + const status = effectiveAlertStatus(selected); + const heading = migrationHeading(selected, status); + const copyFollowUp = () => { + const teamName = params.team ? teamNames.get(params.team) || params.team : "Team owners"; + const count = keyQuery.data?.total_api_keys; + const lastSeen = params.team + ? teamQuery.data?.find((team) => team.id === params.team)?.lastUsedDate ?? selected.lastUsedDate + : selected.lastUsedDate; + const context = { team: teamName, count, lastSeen, url: window.location.href }; + const message = migrationFollowUp(selected, context); + void copyToClipboard(message, "Follow-up copied"); + }; + return ( + <> +
+
+ {status} +

{heading}

+

+ Last seen {selected.lastUsedDate} UTC · {selected.requests.toLocaleString()} requests in the last 30 days +

+ +
+
+ + {!isViewOnly && ( + <> + {status === AlertStatus.Open ? ( + <> + + + + ) : ( + + )} + + )} + {modelGroupHref(selected.model.model_name) && ( + + Model routing + + )} +
+
+

+ Resolve across all teams once owners confirm migration. New traffic reopens the alert. +

+ {statusMutation.isError && ( +

+ Could not save alert status. Please retry. +

+ )} + +
+
+

{params.team ? "Keys in this team" : "Affected keys"}

+

+ {keyQuery.data?.total_api_keys.toLocaleString() ?? "—"} +

+
+
+

Teams using this model

+

{teamQuery.data?.length ?? "—"}

+
+
+

Deprecation

+

+ +

+
+
+
+
+
+

+ Affected keys{" "} + + {keyQuery.data?.total_api_keys.toLocaleString() ?? "…"} + +

+

+ Keys and owners that used this model in the last 30 days. Tags show actual usage with this model. +

+
+ +
+ {keyQuery.isError ? ( +
+

Could not load affected keys.

+ +
+ ) : ( + row.api_key} + /> + )} +
+ {teamQuery.isError && ( +

Could not load team filters. The keys table still includes all matching teams.

+ )} + + ); +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx new file mode 100644 index 00000000000..f8698556d49 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelAlertsPage.integration.test.tsx @@ -0,0 +1,124 @@ +import { beforeEach, afterEach, describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders, screen, testQueryClient, waitFor } from "../../../tests/test-utils"; +import ModelAlertsPage from "@/app/(dashboard)/alerts/page"; + +vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ + useTeams: () => ({ data: [{ team_id: "support", team_alias: "Support" }] }), +})); +vi.mock("@/components/ModelDeprecationAlerts/useDeprecationAlerts", () => ({ + useDeprecationAlerts: () => ({ + scope: "test", + range: { start_date: "2026-10-06", end_date: "2026-11-04" }, + isPending: false, + isError: false, + data: [ + { + id: "retirement", + usageModel: "openai/gpt-4-turbo", + requests: 5000, + lastUsedDate: "2026-11-04", + lastDayRequests: 20, + model: { + model_name: "support-model", + deprecation_date: "2026-10-23", + days_until_deprecation: -12, + status: "deprecated", + successor_model: "replacement-model", + }, + }, + ], + }), +})); + +const calls: { path: string; query: URLSearchParams; method: string; body: unknown }[] = []; +function apiResponse(data: unknown) { + return new Response(JSON.stringify(data), { headers: { "Content-Type": "application/json" } }); +} + +describe("model migration impact", () => { + beforeEach(() => { + testQueryClient.clear(); + calls.length = 0; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + const request = input instanceof Request ? input : new Request(input); + const url = new URL(request.url); + const call: (typeof calls)[number] = { + path: url.pathname, + query: url.searchParams, + method: request.method, + body: request.method === "PUT" ? await request.clone().json() : null, + }; + calls.push(call); + if (url.pathname === "/alerting/model-deprecations/tags") return apiResponse([]); + if (request.method === "PUT" && url.pathname === "/alerting/model-deprecations/state") + return apiResponse({ status: "resolved" }); + if (url.pathname.endsWith("/aggregated/keys")) { + const offset = Number(url.searchParams.get("offset") || 0); + const rows = Array.from({ length: 50 }, (_, i) => ({ + api_key: `reporting-${offset + i}`, + metadata: { key_alias: `support-key-${offset + i}`, team_id: "support", user_email: "owner@example.com" }, + metrics: { api_requests: 100, spend: 1 }, + })); + const page = { api_keys: rows, total_api_keys: 120, offset, limit: 50 }; + return apiResponse(page); + } + if (url.pathname.endsWith("/team/daily/activity/aggregated")) + return apiResponse({ results: [], metadata: { entity_total_api_keys: {} } }); + return apiResponse({}); + }); + }); + afterEach(() => vi.restoreAllMocks()); + it("should render the inbox with last seen and the recommendation supplied by the API", () => { + renderWithProviders(); + expect(screen.getByRole("tab", { name: "Model deprecations", exact: true })).toBeInTheDocument(); + expect(screen.getByText("replacement-model")).toBeInTheDocument(); + expect(screen.getByText("2026-11-04")).toBeInTheDocument(); + expect(screen.getByRole("link", { name: "Review impact →" })).toHaveAttribute( + "href", + "/ui/alerts?type=model-deprecations&model=openai%2Fgpt-4-turbo", + ); + }); + it("should page through all affected keys instead of showing a top-ten sample", async () => { + const user = userEvent.setup(); + renderWithProviders(, { searchParams: { model: "openai/gpt-4-turbo" } }); + expect(await screen.findByRole("link", { name: "support-key-0" })).toBeInTheDocument(); + expect(screen.getByText(/Showing 1-50 of 120/)).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Go to next page", exact: true })); + await screen.findByRole("link", { name: "support-key-50" }); + expect( + calls.some( + (call) => + call.path.endsWith("/aggregated/keys") && + call.query.get("offset") === "50" && + call.query.get("model") === "openai/gpt-4-turbo", + ), + ).toBe(true); + }); + it("should resolve the whole model using its global latest-day snapshot even in a team-filtered view", async () => { + const user = userEvent.setup(); + renderWithProviders(, { + searchParams: { model: "openai/gpt-4-turbo", team: "support" }, + }); + expect(await screen.findByRole("link", { name: "support-key-0" })).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Mark resolved", exact: true })); + const expectedAcknowledgement = { + model: "openai/gpt-4-turbo", + deprecation_date: "2026-10-23", + status: "resolved", + last_seen_date: "2026-11-04", + last_seen_requests: 20, + }; + await waitFor(() => + expect( + calls.find((call) => call.method === "PUT" && call.path === "/alerting/model-deprecations/state")?.body, + ).toEqual(expectedAcknowledgement), + ); + + expect( + calls.some( + (call) => call.path === "/team/daily/activity/aggregated/keys" && call.query.get("team_ids") === "support", + ), + ).toBe(true); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelDeprecationsPanel.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelDeprecationsPanel.tsx new file mode 100644 index 00000000000..ef19bdea368 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelDeprecationsPanel.tsx @@ -0,0 +1,76 @@ +"use client"; + +import { useQueryStates } from "nuqs"; +import { ProviderLogo } from "@/components/molecules/models/ProviderLogo"; +import { deprecationProvider } from "./DeprecationModel"; +import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; +import { Page } from "@/components/shared/Page"; +import { Button, buttonVariants } from "@/components/ui/button"; +import { useDeprecationAlerts } from "@/components/ModelDeprecationAlerts/useDeprecationAlerts"; +import { hasCapability } from "@/utils/capabilities"; +import { migrationUrlParsers, migrationHref } from "./migrationRoute"; +import AlertsInbox from "./AlertsInbox"; +import MigrationDetail from "./MigrationDetail"; + +export default function ModelDeprecationsPanel() { + const { accessToken, userRole, isViewOnly } = useAuthorized(); + const allowed = hasCapability(userRole, "viewGlobalSpend"); + const [params, setParams] = useQueryStates(migrationUrlParsers); + const alerts = useDeprecationAlerts(accessToken); + const selected = alerts.data?.find((alert) => alert.usageModel === params.model); + const loadingDetail = !alerts.isError && Boolean(params.model) && alerts.isPending; + const detailReady = !alerts.isError && Boolean(params.model) && !alerts.isPending; + const missingDetail = detailReady && !selected; + if (!allowed) + return ( + +

Model deprecations

+

Gateway admin access is required to view affected keys and teams.

+ + ); + return ( + + {params.model && ( +
+
+

+ {selected && } + + {params.model} + +

+
+ + ← Model deprecations + +
+ )} + {alerts.isError ? ( +
+

Could not load alerts. No alert status has been changed.

+ +
+ ) : null} + {!alerts.isError && !params.model && ( + void alerts.refetch()} + refreshing={alerts.isFetching} + status={params.state} + onStatusChange={(state) => void setParams({ state })} + /> + )} + {loadingDetail &&

Loading alert…

} + {missingDetail &&

No matching model alert was found in the last 30 days.

} + {!alerts.isError && selected && ( + + )} + + ); +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelUsageChart.integration.test.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelUsageChart.integration.test.tsx new file mode 100644 index 00000000000..2ca03e64f1d --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelUsageChart.integration.test.tsx @@ -0,0 +1,103 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import userEvent from "@testing-library/user-event"; +import { renderWithProviders, screen } from "../../../tests/test-utils"; +import ModelUsageChart from "./ModelUsageChart"; + +vi.mock("@/components/shared/charts", async (importOriginal) => ({ + ...(await importOriginal()), + StackedUsageChart: ({ data, labels }: { data: unknown; labels: string[] }) => ( +
+ {JSON.stringify({ data, labels })} +
+ ), +})); +const range = { start_date: "2026-10-07", end_date: "2026-11-05" }; +function mockActivity() { + return vi.spyOn(globalThis, "fetch").mockImplementation( + async () => + new Response( + JSON.stringify({ + results: [ + { + date: "2026-11-03", + metrics: { api_requests: 100, total_tokens: 1000, spend: 10 }, + breakdown: { + api_keys: { + "key-a": { + metrics: { api_requests: 30, spend: 3, total_tokens: 300 }, + metadata: { key_alias: "Support key" }, + }, + "key-b": { + metrics: { api_requests: 20, spend: 2, total_tokens: 200 }, + metadata: { key_alias: "Research key" }, + }, + }, + }, + }, + ], + }), + { headers: { "Content-Type": "application/json" } }, + ), + ); +} + +describe("model daily usage", () => { + afterEach(() => vi.restoreAllMocks()); + it("should render the model's daily requests stacked by key and preserve remaining usage", async () => { + const fetch = mockActivity(); + renderWithProviders( + , + ); + const chart = await screen.findByRole("img", { name: "Daily key usage chart" }); + expect(chart).toHaveTextContent('"labels":["Support key","Research key","Other"]'); + expect(chart).toHaveTextContent('"s0":30,"s1":20,"s2":50'); + const request = fetch.mock.calls[0][0] as Request; + expect(new URL(request.url).searchParams.get("model")).toBe("anthropic/retiring-model"); + }); + it("should apply the selected team and show requests without metric controls", async () => { + const fetch = mockActivity(); + renderWithProviders( + , + ); + await screen.findByRole("img", { name: "Daily key usage chart" }); + const request = fetch.mock.calls[0][0] as Request; + expect(new URL(request.url).pathname).toBe("/team/daily/activity/aggregated"); + expect(new URL(request.url).searchParams.get("team_ids")).toBe("support"); + expect(screen.queryByRole("radio")).not.toBeInTheDocument(); + expect(screen.getByRole("img", { name: "Daily key usage chart" })).toHaveTextContent('"s0":30,"s1":20,"s2":50'); + }); + it("should keep all keys in the stacks while allowing the key labels to expand", async () => { + const fetch = mockActivity(); + const keys = Object.fromEntries( + Array.from({ length: 12 }, (_, index) => [ + `key-${index}`, + { + metrics: { api_requests: 10, spend: 1, total_tokens: 100 }, + metadata: { key_alias: `Key ${index + 1}` }, + }, + ]), + ); + const response = { + results: [ + { + date: "2026-11-03", + metrics: { api_requests: 120, spend: 12, total_tokens: 1200 }, + breakdown: { api_keys: keys }, + }, + ], + }; + fetch.mockResolvedValueOnce( + new Response(JSON.stringify(response), { headers: { "Content-Type": "application/json" } }), + ); + const user = userEvent.setup(); + renderWithProviders( + , + ); + expect(await screen.findByRole("img", { name: "Daily key usage chart" })).toHaveTextContent('"s11":10'); + expect(screen.queryByText("Key 12", { exact: true })).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Show all key labels" })); + expect(screen.getByText("Key 12", { exact: true })).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Show fewer key labels" })); + expect(screen.queryByText("Key 12", { exact: true })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelUsageChart.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelUsageChart.tsx new file mode 100644 index 00000000000..7f8262c83fd --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/ModelUsageChart.tsx @@ -0,0 +1,127 @@ +"use client"; + +import { useMemo, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { StackedUsageChart } from "@/components/shared/charts"; +import { Button } from "@/components/ui/button"; +import { backgroundFetchClient } from "@/lib/http/api"; +import { toDailyData } from "@/components/UsagePage/dailyActivityApi"; +import { Panel, ChartSkeleton } from "@/app/(dashboard)/usage/_components/components/overview/Primitives"; +import { seriesBy } from "@/app/(dashboard)/usage/_components/components/overview/overviewData"; +import type { deprecationUsageRange } from "./deprecationAlerts"; +import { dailyModelRequests } from "./migrationData"; + +const formatDate = (date: string) => + new Date(`${date}T00:00:00Z`).toLocaleDateString("en-US", { month: "short", day: "numeric", timeZone: "UTC" }); + +export default function ModelUsageChart({ + model, + team, + scope, + range, +}: { + model: string; + team: string | null; + scope: string; + range: ReturnType; +}) { + const [legendExpanded, setLegendExpanded] = useState(false); + const activityOptions = { + queryKey: ["deprecationDailyUsage", scope, model, team, range], + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const query = { ...range, model, api_key_limit: 100, timezone: 0 }; + if (team) { + const { data } = await backgroundFetchClient.GET("/team/daily/activity/aggregated", { + params: { query: { ...query, team_ids: team } }, + signal, + }); + if (!data) throw new Error("Daily model usage is unavailable"); + return data; + } + const { data } = await backgroundFetchClient.GET("/user/daily/activity/aggregated", { + params: { query }, + signal, + }); + if (!data) throw new Error("Daily model usage is unavailable"); + return data; + }, + staleTime: 5 * 60 * 1000, + refetchInterval: 5 * 60 * 1000, + }; + const activity = useQuery(activityOptions); + const daily = useMemo(() => dailyModelRequests(activity.data?.results ?? [], range), [activity.data, range]); + const totals = useMemo(() => new Map(daily.map((day) => [day.date, day.requests])), [daily]); + const series = useMemo(() => { + const days = activity.data ? toDailyData(activity.data) : []; + const aliases = new Map( + days.flatMap((day) => + Object.entries(day.breakdown.api_keys).map( + ([id, entry]) => [id, entry.metadata.key_alias || `Key · ${id.slice(0, 6)}`] as const, + ), + ), + ); + const result = seriesBy(days, "api_keys", "requests", 100); + const byDate = new Map(result.data.map((day) => [day.date, day])); + const zeros = Object.fromEntries(result.keys.map((key) => [key, 0])); + result.data = daily.map((day) => byDate.get(day.date) ?? { date: day.date, label: formatDate(day.date), ...zeros }); + result.labels = result.labels.map((label) => aliases.get(label) || label); + return result; + }, [activity.data, daily]); + function renderContent() { + if (activity.isPending) return ; + if (activity.isError) + return ( +
+ Could not load daily requests.{" "} + +
+ ); + return ( + <> + value.toLocaleString()} + totalFor={(date) => totals.get(date)} + className="h-64" + /> +
+ {series.labels.slice(0, legendExpanded ? undefined : 8).map((label, index) => ( + + + {label} + + ))} +
+ {series.labels.length > 8 && ( + + )} + + ); + } + return ( + + {renderContent()} +

+ Last 30 days · UTC · Includes zero-usage days. Today is partial. +

+ + ); +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/RecommendedReplacement.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/RecommendedReplacement.tsx new file mode 100644 index 00000000000..e1c237e6ff8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/RecommendedReplacement.tsx @@ -0,0 +1,7 @@ +export default function RecommendedReplacement({ model }: { model: string | null | undefined }) { + return ( +

+ Recommended new model: {model || "Not provided"} +

+ ); +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/UsageTags.test.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/UsageTags.test.tsx new file mode 100644 index 00000000000..9d1767bb76b --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/UsageTags.test.tsx @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import UsageTags from "./UsageTags"; + +describe("model usage tags", () => { + it("should render tags used with the model", () => { + render(); + expect(screen.getByText("production")).toBeInTheDocument(); + }); + it("should show five tags and let the user expand and collapse the rest", async () => { + const user = userEvent.setup(); + render(); + expect(screen.queryByText("six")).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Show 2 more usage tags" })); + expect(screen.getByText("six")).toBeInTheDocument(); + expect(screen.getByText("seven")).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Show fewer usage tags" })); + expect(screen.queryByText("six")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/UsageTags.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/UsageTags.tsx new file mode 100644 index 00000000000..a25ef5ec5b3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/UsageTags.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { useState } from "react"; +import { Badge } from "@/components/ui/badge"; + +export default function UsageTags({ + tags, + loading, + failed, +}: { + tags: readonly string[]; + loading: boolean; + failed: boolean; +}) { + const [expanded, setExpanded] = useState(false); + if (loading) return Loading tags…; + if (failed) return Tags unavailable; + if (tags.length === 0) return No recorded tags; + const visible = expanded ? tags : tags.slice(0, 5); + return ( +
+ {visible.map((tag) => ( + + {tag} + + ))} + {tags.length > 5 && ( + + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.test.ts b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.test.ts new file mode 100644 index 00000000000..f38eee7fccf --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import type { components } from "@/lib/http/schema"; +import { deprecationUsageRange, usedDeprecationAlerts } from "./deprecationAlerts"; + +type Model = components["schemas"]["ModelDeprecationInfo"]; +type Day = components["schemas"]["DailySpendData"]; +const model: Model = { + model_name: "support", + litellm_model: "anthropic/claude-sonnet-4-5", + deprecation_date: "2026-11-30", + days_until_deprecation: 26, + status: "imminent", +}; +// Use the API's defaults for the unrelated aggregate metrics. +const metrics: components["schemas"]["SpendMetrics"] = { + spend: 0, + api_requests: 0, + successful_requests: 0, + failed_requests: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}; +function day(date: string, count: number, name = model.litellm_model!): Day { + return { date, metrics, breakdown: { models: { [name]: { metrics: { ...metrics, api_requests: count } } } } }; +} + +describe("usedDeprecationAlerts", () => { + it("should aggregate requests and report the latest day of actual model usage", () => { + const result = usedDeprecationAlerts([model], [day("2026-11-04", 12), day("2026-10-20", 8)]); + expect(result).toHaveLength(1); + expect(result[0]).toMatchObject({ usageModel: model.litellm_model, requests: 20, lastUsedDate: "2026-11-04" }); + }); + it("should omit configured models without observed requests", () => { + expect(usedDeprecationAlerts([model], [])).toEqual([]); + expect(usedDeprecationAlerts([model], [day("2026-11-04", 0)])).toEqual([]); + }); + it("should avoid attributing healthy deployments in the same group to a deprecated model", () => { + const activity = day("2026-11-04", 100, "anthropic/claude-sonnet-4-6"); + activity.breakdown!.model_groups = { support: { metrics: { ...metrics, api_requests: 100 } } }; + expect(usedDeprecationAlerts([model], [activity])).toEqual([]); + }); + it("should match provider-qualified usage when the cost map names an unqualified model", () => { + const entry = { ...model, litellm_model: "claude-sonnet-4-5", litellm_provider: "anthropic" }; + const result = usedDeprecationAlerts([entry], [day("2026-11-04", 12)]); + expect(result[0]).toMatchObject({ usageModel: "anthropic/claude-sonnet-4-5", requests: 12 }); + }); + it("should keep usage for different providers separate", () => { + const entry = { ...model, litellm_provider: "anthropic" }; + expect(usedDeprecationAlerts([entry], [day("2026-11-04", 12, "bedrock/claude-sonnet-4-5")])).toEqual([]); + }); + it("should deduplicate deployments and put the earliest retirement first", () => { + const retired = { ...model, deprecation_date: "2026-10-23", status: "deprecated" as const }; + const result = usedDeprecationAlerts([model, retired, model], [day("2026-11-04", 10)]); + expect(result.map((alert) => alert.model.deprecation_date)).toEqual(["2026-10-23", "2026-11-30"]); + }); + it("should give an escalated alert a new dismissal identity", () => { + const activity = [day("2026-11-04", 10)]; + const imminent = usedDeprecationAlerts([model], activity)[0]; + const retired = usedDeprecationAlerts([{ ...model, status: "deprecated" }], activity)[0]; + expect(retired.id).not.toBe(imminent.id); + }); + it("should use the public model name when underlying model metadata is unavailable", () => { + const result = usedDeprecationAlerts([{ ...model, litellm_model: null }], [day("2026-11-04", 10, "support")]); + expect(result[0].requests).toBe(10); + }); +}); + +describe("deprecationUsageRange", () => { + it("should include exactly 30 UTC calendar dates across month boundaries", () => { + expect(deprecationUsageRange(new Date("2026-11-04T23:00:00Z"))).toEqual({ + start_date: "2026-10-06", + end_date: "2026-11-04", + }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.ts b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.ts new file mode 100644 index 00000000000..d22d940ab16 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/deprecationAlerts.ts @@ -0,0 +1,61 @@ +import type { components } from "@/lib/http/schema"; + +type ModelDeprecation = components["schemas"]["ModelDeprecationInfo"]; +type DailySpendData = components["schemas"]["DailySpendData"]; + +export interface DeprecationAlert { + id: string; + model: ModelDeprecation; + usageModel: string; + requests: number; + lastUsedDate: string; + lastDayRequests: number; + state?: components["schemas"]["DeprecationAlertState"]; +} + +export function deprecationUsageRange(today: Date) { + const end = new Date(today); + const start = new Date(today); + start.setUTCDate(start.getUTCDate() - 29); + return { start_date: start.toISOString().slice(0, 10), end_date: end.toISOString().slice(0, 10) }; +} + +/** Match actual model usage, rather than model-group totals that may include healthy deployments. */ +export function usedDeprecationAlerts( + models: readonly ModelDeprecation[], + days: readonly DailySpendData[], +): DeprecationAlert[] { + const alerts: DeprecationAlert[] = []; + const seen = new Set(); + for (const model of models) { + const baseModel = model.litellm_model ?? model.model_name; + const candidates = new Set([baseModel]); + if (model.litellm_provider && !baseModel.startsWith(`${model.litellm_provider}/`)) { + candidates.add(`${model.litellm_provider}/${baseModel}`); + } + for (const usageModel of candidates) { + const id = JSON.stringify([usageModel, model.deprecation_date, model.status]); + if (seen.has(id)) continue; + seen.add(id); + let requests = 0; + let lastUsedDate = ""; + let lastDayRequests = 0; + for (const day of days) { + const metrics = day.breakdown?.models?.[usageModel]?.metrics; + if (!metrics || metrics.api_requests <= 0) continue; + requests += metrics.api_requests; + if (day.date > lastUsedDate) { + lastUsedDate = day.date; + lastDayRequests = metrics.api_requests; + } else if (day.date === lastUsedDate) { + lastDayRequests += metrics.api_requests; + } + } + if (requests > 0) { + const alert = { id, model, usageModel, requests, lastUsedDate, lastDayRequests }; + alerts.push(alert); + } + } + } + return alerts.sort((a, b) => a.model.deprecation_date.localeCompare(b.model.deprecation_date)); +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationData.test.ts b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationData.test.ts new file mode 100644 index 00000000000..17a5a47391f --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationData.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import type { components } from "@/lib/http/schema"; +import { dailyModelRequests } from "./migrationData"; + +type Day = components["schemas"]["DailySpendData"]; +function day(date: string, api_requests: number): Day { + return { + date, + metrics: { + api_requests, + spend: 0, + prompt_tokens: 0, + completion_tokens: 0, + total_tokens: 0, + successful_requests: api_requests, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, + }, + }; +} + +describe("model request trend", () => { + it("should include zero-usage days after the model stops receiving traffic", () => { + expect(dailyModelRequests([day("2026-11-03", 120)], { start_date: "2026-11-03", end_date: "2026-11-05" })).toEqual([ + { date: "2026-11-03", requests: 120 }, + { date: "2026-11-04", requests: 0 }, + { date: "2026-11-05", requests: 0 }, + ]); + }); + it("should keep dates chronological across month boundaries and preserve daily totals", () => { + expect( + dailyModelRequests([day("2026-11-01", 5), day("2026-10-31", 50)], { + start_date: "2026-10-31", + end_date: "2026-11-02", + }), + ).toEqual([ + { date: "2026-10-31", requests: 50 }, + { date: "2026-11-01", requests: 5 }, + { date: "2026-11-02", requests: 0 }, + ]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationData.ts b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationData.ts new file mode 100644 index 00000000000..99b97c0fa3a --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationData.ts @@ -0,0 +1,49 @@ +import type { components } from "@/lib/http/schema"; + +type Activity = components["schemas"]["SpendAnalyticsPaginatedResponse"]; +export interface AffectedTeam { + id: string; + requests: number; + keys: number; + lastUsedDate: string; +} + +export function affectedTeams(activity: Activity): AffectedTeam[] { + const teams = new Map(); + for (const day of activity.results ?? []) { + for (const [id, entry] of Object.entries(day.breakdown?.entities ?? {})) { + if (entry.metrics.api_requests <= 0) continue; + const team = teams.get(id) ?? { + id, + requests: 0, + keys: activity.metadata?.entity_total_api_keys?.[id] ?? 0, + lastUsedDate: "", + }; + team.requests += entry.metrics.api_requests; + if (day.date > team.lastUsedDate) team.lastUsedDate = day.date; + teams.set(id, team); + } + } + return [...teams.values()].sort((a, b) => b.requests - a.requests); +} + +export interface DailyModelRequests extends Record { + date: string; + requests: number; +} + +/** Include quiet days so a completed migration remains visible as zero traffic. */ +export function dailyModelRequests( + days: readonly components["schemas"]["DailySpendData"][], + range: { start_date: string; end_date: string }, +): DailyModelRequests[] { + const counts = new Map(); + for (const day of days) counts.set(day.date, (counts.get(day.date) ?? 0) + day.metrics.api_requests); + const result: DailyModelRequests[] = []; + const end = new Date(`${range.end_date}T00:00:00Z`); + for (const day = new Date(`${range.start_date}T00:00:00Z`); day <= end; day.setUTCDate(day.getUTCDate() + 1)) { + const date = day.toISOString().slice(0, 10); + result.push({ date, requests: counts.get(date) ?? 0 }); + } + return result; +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationRoute.ts b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationRoute.ts new file mode 100644 index 00000000000..4581f8042f4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationRoute.ts @@ -0,0 +1,14 @@ +import { parseAsString, parseAsStringEnum } from "nuqs"; +import { AlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; +import { uiHref } from "@/utils/uiHref"; + +export const migrationUrlParsers = { + model: parseAsString, + team: parseAsString, + state: parseAsStringEnum(Object.values(AlertStatus)).withDefault(AlertStatus.Open), +}; + +export function migrationHref(model?: string) { + const base = `${uiHref("alerts")}?type=model-deprecations`; + return model ? `${base}&model=${encodeURIComponent(model)}` : base; +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationState.test.ts b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationState.test.ts new file mode 100644 index 00000000000..63581056ece --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationState.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "vitest"; +import { AlertStatus, effectiveAlertStatus } from "./migrationState"; +import type { DeprecationAlert } from "./deprecationAlerts"; +const alert: DeprecationAlert = { + id: "model", + usageModel: "openai/gpt-4-turbo", + requests: 1000, + lastUsedDate: "2026-11-04", + lastDayRequests: 10, + model: { model_name: "support", deprecation_date: "2026-10-23", days_until_deprecation: -12, status: "deprecated" }, +}; +const state = { + model: alert.usageModel, + deprecation_date: "2026-10-23", + status: "resolved" as const, + last_seen_date: "2026-11-04", + last_seen_requests: 10, +}; +describe("effectiveAlertStatus", () => { + it("should keep a resolved migration closed when only its historic traffic remains", () => { + expect(effectiveAlertStatus({ ...alert, state })).toBe(AlertStatus.Resolved); + }); + it("should reopen on additional requests in the same UTC day", () => { + expect(effectiveAlertStatus({ ...alert, lastDayRequests: 11, state })).toBe(AlertStatus.Open); + }); + it("should reopen when the model is used on a later day", () => { + expect(effectiveAlertStatus({ ...alert, lastUsedDate: "2026-11-05", state })).toBe(AlertStatus.Open); + }); + it("should keep acknowledgement when old requests roll out of the 30-day window", () => { + expect(effectiveAlertStatus({ ...alert, requests: 900, state })).toBe(AlertStatus.Resolved); + }); + it("should allow a shared dismissed state and reopen it on fresh traffic", () => { + const dismissed = { ...state, status: "dismissed" as const }; + expect(effectiveAlertStatus({ ...alert, state: dismissed })).toBe(AlertStatus.Dismissed); + expect(effectiveAlertStatus({ ...alert, lastDayRequests: 11, state: dismissed })).toBe(AlertStatus.Open); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationState.ts b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationState.ts new file mode 100644 index 00000000000..006cea4a985 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/migrationState.ts @@ -0,0 +1,17 @@ +import type { DeprecationAlert } from "./deprecationAlerts"; + +export enum AlertStatus { + Open = "open", + Resolved = "resolved", + Dismissed = "dismissed", +} + +export function effectiveAlertStatus(alert: DeprecationAlert): AlertStatus { + const state = alert.state; + if (!state || state.status === "open") return AlertStatus.Open; + if (!state.last_seen_date || alert.lastUsedDate > state.last_seen_date) return AlertStatus.Open; + if (alert.lastUsedDate === state.last_seen_date && alert.lastDayRequests > state.last_seen_requests) + return AlertStatus.Open; + if (state.status === "resolved") return AlertStatus.Resolved; + return AlertStatus.Dismissed; +} diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx new file mode 100644 index 00000000000..897850393e2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.integration.test.tsx @@ -0,0 +1,74 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { renderWithProviders, screen, testQueryClient } from "../../../tests/test-utils"; +import { backgroundFetchClient } from "@/lib/http/api"; +import { useDeprecationAlerts } from "./useDeprecationAlerts"; +import { useCurrentUser } from "@/app/(dashboard)/hooks/currentUser/useCurrentUser"; + +vi.mock("@/lib/http/api", () => ({ backgroundFetchClient: { GET: vi.fn() } })); +vi.mock("@/app/(dashboard)/hooks/currentUser/useCurrentUser", () => ({ useCurrentUser: vi.fn() })); + +function AlertsHost() { + const { data, isError, fetchStatus } = useDeprecationAlerts("test-session"); + if (isError) return
Unavailable
; + if (fetchStatus !== "idle") return
Loading
; + return
{data?.length ?? 0} alerts
; +} + +describe("deprecation alert API wiring", () => { + beforeEach(() => { + vi.clearAllMocks(); + testQueryClient.clear(); + }); + it("should avoid querying gateway-wide usage for non-admins", () => { + vi.mocked(useCurrentUser).mockReturnValue({ + data: { user_id: "internal", user_role: "internal_user" }, + } as ReturnType); + renderWithProviders(); + expect(screen.getByText("0 alerts")).toBeInTheDocument(); + expect(backgroundFetchClient.GET).not.toHaveBeenCalled(); + }); + it("should skip the usage query when no configured models need attention", async () => { + vi.mocked(useCurrentUser).mockReturnValue({ data: { user_id: "admin", user_role: "proxy_admin" } } as ReturnType< + typeof useCurrentUser + >); + vi.mocked(backgroundFetchClient.GET).mockResolvedValue({ + data: { deprecated: [], imminent: [], upcoming: [] }, + } as never); + renderWithProviders(); + expect(await screen.findByText("0 alerts")).toBeInTheDocument(); + expect(backgroundFetchClient.GET).toHaveBeenCalledTimes(1); + expect(backgroundFetchClient.GET).toHaveBeenCalledWith( + "/model/deprecations", + expect.objectContaining({ params: { query: { warn_within_days: 30 } } }), + ); + }); + it("should load acknowledgement state from the alerting API for models with observed usage", async () => { + vi.mocked(useCurrentUser).mockReturnValue({ data: { user_id: "admin", user_role: "proxy_admin" } } as ReturnType< + typeof useCurrentUser + >); + const model = { + model_name: "support", + litellm_model: "anthropic/claude-sonnet-4-5", + deprecation_date: "2026-11-30", + status: "imminent", + }; + const schedule = { deprecated: [], imminent: [model], upcoming: [] }; + const activity = { + results: [ + { date: "2026-11-04", breakdown: { models: { [model.litellm_model]: { metrics: { api_requests: 10 } } } } }, + ], + }; + vi.mocked(backgroundFetchClient.GET) + .mockResolvedValueOnce({ data: schedule } as never) + .mockResolvedValueOnce({ data: activity } as never) + .mockResolvedValueOnce({ data: { status: "open" } } as never); + renderWithProviders(); + expect(await screen.findByText("1 alerts")).toBeInTheDocument(); + expect(backgroundFetchClient.GET).toHaveBeenLastCalledWith( + "/alerting/model-deprecations/state", + expect.objectContaining({ + params: { query: { model: model.litellm_model, deprecation_date: model.deprecation_date } }, + }), + ); + }); +}); diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts new file mode 100644 index 00000000000..ce126d5cd4f --- /dev/null +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/useDeprecationAlerts.ts @@ -0,0 +1,51 @@ +import { useQuery } from "@tanstack/react-query"; +import { useCurrentUser } from "@/app/(dashboard)/hooks/currentUser/useCurrentUser"; +import { backgroundFetchClient } from "@/lib/http/api"; +import { getProxyBaseUrl } from "@/components/networking"; +import { hasCapability } from "@/utils/capabilities"; +import { deprecationUsageRange, usedDeprecationAlerts } from "./deprecationAlerts"; + +const POLL_INTERVAL = 5 * 60 * 1000; + +export function useDeprecationAlerts(accessToken: string | null) { + const { data: currentUser } = useCurrentUser(accessToken); + const userId = currentUser?.user_id; + const enabled = Boolean(accessToken && userId && hasCapability(currentUser?.user_role, "viewGlobalSpend")); + const scope = JSON.stringify([getProxyBaseUrl(), userId]); + const range = deprecationUsageRange(new Date()); + const options = { + queryKey: ["deprecationAlerts", scope, range], + enabled, + queryFn: async ({ signal }: { signal: AbortSignal }) => { + const { data: schedule } = await backgroundFetchClient.GET("/model/deprecations", { + params: { query: { warn_within_days: 30 } }, + signal, + }); + if (!schedule) throw new Error("Model deprecation information is unavailable"); + const models = [...(schedule.deprecated ?? []), ...(schedule.imminent ?? [])]; + if (models.length === 0) return []; + const { data: activity } = await backgroundFetchClient.GET("/user/daily/activity/aggregated", { + params: { query: { ...range, timezone: 0, api_key_limit: 1 } }, + signal, + }); + if (!activity) throw new Error("Model usage is unavailable"); + const alerts = usedDeprecationAlerts(models, activity.results ?? []); + return Promise.all( + alerts.map(async (alert) => { + const { data: state } = await backgroundFetchClient.GET("/alerting/model-deprecations/state", { + params: { query: { model: alert.usageModel, deprecation_date: alert.model.deprecation_date } }, + signal, + }); + if (!state) throw new Error("Alert acknowledgement state is unavailable"); + alert.state = state; + return alert; + }), + ); + }, + staleTime: POLL_INTERVAL, + refetchInterval: POLL_INTERVAL, + retry: false, + }; + const query = useQuery(options); + return { ...query, data: enabled ? query.data : undefined, scope, range }; +} diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx index 93d9b6018ac..3b0eec6feec 100644 --- a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.test.tsx @@ -5,6 +5,14 @@ import React from "react"; import userEvent from "@testing-library/user-event"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +vi.mock("@/components/ModelDeprecationAlerts/useDeprecationAlerts", () => ({ + useDeprecationAlerts: () => ({ + data: [], + scope: "test", + range: { start_date: "2026-10-06", end_date: "2026-11-04" }, + }), +})); + vi.mock("@/app/(dashboard)/hooks/healthReadiness/useHealthReadinessDetails", () => ({ useHealthReadinessDetails: vi.fn(), })); @@ -173,3 +181,77 @@ describe("NotificationsBell", () => { expect(announcement("LiteLLM v1.103.0 is available")).toHaveTextContent("Your current version is v1.102.0"); }); }); + +const DEPRECATION = { + id: "sonnet:2026-11-30:imminent", + usageModel: "anthropic/claude-sonnet-4-5", + requests: 18432, + lastUsedDate: "2026-11-04", + lastDayRequests: 123, + model: { + model_name: "support", + litellm_model: "anthropic/claude-sonnet-4-5", + deprecation_date: "2026-11-30", + days_until_deprecation: 26, + status: "imminent" as const, + successor_model: "recommended-model", + }, +}; + +describe("model deprecation notifications", () => { + beforeEach(() => localStorage.clear()); + afterEach(() => localStorage.clear()); + it("should show last seen and link to the migration workspace and open alerts inbox", async () => { + const user = userEvent.setup(); + renderWithProviders( + , + ); + await user.click(screen.getByRole("button", { name: /1 model deprecation alert/ })); + expect(screen.getByRole("article", { name: /retires in 26 days/ })).toHaveTextContent("Last seen 2026-11-04 UTC"); + expect(screen.getByRole("link", { name: "Review migration" })).toHaveAttribute( + "href", + "/ui/alerts?type=model-deprecations&model=anthropic%2Fclaude-sonnet-4-5", + ); + expect(screen.getByRole("link", { name: "View all 1" })).toHaveAttribute( + "href", + "/ui/alerts?type=model-deprecations", + ); + expect(screen.getByText("recommended-model")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Dismiss", exact: true })).not.toBeInTheDocument(); + }); + it("should clear the warning when the alert has a shared resolved acknowledgement", () => { + const state = { + model: DEPRECATION.usageModel, + deprecation_date: "2026-11-30", + status: "resolved" as const, + last_seen_date: "2026-11-04", + last_seen_requests: 123, + }; + renderWithProviders( + , + ); + expect(bell(1)).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: /model deprecation alert/ })).not.toBeInTheDocument(); + }); + it("should reopen a resolved alert when more requests arrive on the last-seen day", () => { + const state = { + model: DEPRECATION.usageModel, + deprecation_date: "2026-11-30", + status: "resolved" as const, + last_seen_date: "2026-11-04", + last_seen_requests: 122, + }; + renderWithProviders( + , + ); + expect(screen.getByRole("button", { name: /1 model deprecation alert/ })).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx index 2c26f34b750..2756af0e60b 100644 --- a/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx +++ b/ui/litellm-dashboard/src/components/Navbar/NotificationsBell/NotificationsBell.tsx @@ -17,7 +17,12 @@ import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/compone import { Separator } from "@/components/ui/separator"; import { cn } from "@/lib/cva.config"; import { isNewerVersion } from "@/utils/versionUtils"; -import { ArrowUpRight, Bell } from "lucide-react"; +import { AlertTriangle, ArrowUpRight, Bell } from "lucide-react"; +import DeprecationAlertItem from "@/components/ModelDeprecationAlerts/DeprecationAlertItem"; +import { type DeprecationAlert } from "@/components/ModelDeprecationAlerts/deprecationAlerts"; +import { useDeprecationAlerts } from "@/components/ModelDeprecationAlerts/useDeprecationAlerts"; +import { AlertStatus, effectiveAlertStatus } from "@/components/ModelDeprecationAlerts/migrationState"; +import { migrationHref } from "@/components/ModelDeprecationAlerts/migrationRoute"; import React, { Fragment } from "react"; import { useTranslation } from "react-i18next"; import i18n from "@/i18n/i18n"; @@ -102,6 +107,7 @@ const AnnouncementItem: React.FC<{ announcement: Announcement }> = ({ announceme }; interface NotificationsBellViewProps { + deprecationAlerts?: readonly DeprecationAlert[]; deploymentHealthLink?: React.ReactNode; currentVersion: string | null | undefined; latestRelease: LatestReleaseInfo | null | undefined; @@ -111,25 +117,39 @@ export const NotificationsBellView: React.FC = ({ currentVersion, latestRelease, deploymentHealthLink, + deprecationAlerts, }) => { const { t } = useTranslation(); const upgrade = upgradeAnnouncement(currentVersion, latestRelease); const announcements = upgrade ? [upgrade, AUTO_ROUTER_ANNOUNCEMENT] : [AUTO_ROUTER_ANNOUNCEMENT]; const unreadCount = useUnreadAnnouncementCount(announcements.map((announcement) => announcement.readKey)); + const openAlerts = (deprecationAlerts ?? []).filter((alert) => effectiveAlertStatus(alert) === AlertStatus.Open); + const openDeprecations = openAlerts.length; + let label = + unreadCount > 0 + ? t("shell.notifications.bell.triggerUnread", { unread: unreadCount }) + : t("shell.notifications.bell.trigger"); + if (openDeprecations > 0) { + label = `Notifications, ${openDeprecations} model deprecation ${openDeprecations === 1 ? "alert needs" : "alerts need"} attention`; + } + return ( } - aria-label={ - unreadCount > 0 - ? t("shell.notifications.bell.triggerUnread", { unread: unreadCount }) - : t("shell.notifications.bell.trigger") - } + aria-label={label} > - - {unreadCount > 0 ? : null} + 0 && "text-amber-700")} aria-hidden /> + {openDeprecations > 0 ? ( + + + + ) : null} + {openDeprecations === 0 && unreadCount > 0 ? ( + + ) : null} = ({
+ {openAlerts.length > 0 && ( + <> +
+
+

Model deprecations

+ + View all {openDeprecations} + +
+

Model migrations across your keys and teams.

+
+ {openAlerts.slice(0, 3).map((alert) => ( + + + + + ))} + +

Announcements

+ + )} + {deprecationAlerts && openAlerts.length === 0 && ( + + View model deprecations + + )} + {announcements.map((announcement, index) => ( {index > 0 ? : null} @@ -162,10 +209,12 @@ export const NotificationsBell: React.FC<{ accessToken: string | null; deploymen }) => { const { data: healthData } = useHealthReadinessDetails(accessToken); const { data: latestRelease } = useLatestReleaseInfo(accessToken); + const { data: deprecationAlerts } = useDeprecationAlerts(accessToken); return ( ); diff --git a/ui/litellm-dashboard/src/components/leftnav.test.tsx b/ui/litellm-dashboard/src/components/leftnav.test.tsx index 1c44d4dae6d..bab51aab121 100644 --- a/ui/litellm-dashboard/src/components/leftnav.test.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.test.tsx @@ -693,6 +693,7 @@ describe("Sidebar (leftnav)", () => { expectHref("Playground", "/ui/playground"); expectHref("Models + Endpoints", "/ui/models-and-endpoints"); expectHref("Usage", "/ui/usage"); + expectHref("Alerts", "/ui/alerts"); expectHref("API Reference", "/ui/api-reference"); }); diff --git a/ui/litellm-dashboard/src/components/leftnav.tsx b/ui/litellm-dashboard/src/components/leftnav.tsx index 488113d726e..33113b4ad6f 100644 --- a/ui/litellm-dashboard/src/components/leftnav.tsx +++ b/ui/litellm-dashboard/src/components/leftnav.tsx @@ -299,6 +299,13 @@ const menuGroups: MenuGroup[] = [ roles: [...all_admin_roles, ...internalUserRoles], label: "Usage", }, + { + key: "alerts", + page: "alerts", + label: "Alerts", + icon: , + roles: rolesWithCapability("viewGlobalSpend"), + }, { key: "model-insights", page: "model-insights", diff --git a/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx b/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx index 82221826a68..a42db06d251 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.integration.test.tsx @@ -1,10 +1,9 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; -import { fireEvent, render, screen, waitFor, within } from "@testing-library/react"; +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import React, { ReactNode } from "react"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; import ModelInfoView from "./model_info_view"; -import ModelDeprecationBanner from "@/app/(dashboard)/models-and-endpoints/components/ModelDeprecationBanner"; import { toast } from "@/lib/toast"; import * as networking from "./networking"; import { useUserLookup } from "@/app/(dashboard)/hooks/users/useUsers"; @@ -46,10 +45,6 @@ vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ useModelCostMap: (...args: any[]) => mockUseModelCostMap(...args), })); -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ - default: () => ({ accessToken: "test-token", userRole: "Admin" }), -})); - const mockUseTeams = vi.fn(); vi.mock("@/app/(dashboard)/hooks/teams/useTeams", () => ({ useTeams: () => mockUseTeams(), @@ -82,8 +77,6 @@ const mockModelDeleteCall = vi.mocked(networking.modelDeleteCall); const mockCredentialCreateCall = vi.mocked(networking.credentialCreateCall); const mockVectorStoreListCall = vi.mocked(networking.vectorStoreListCall); -const TEST_TODAY = new Date(2026, 10, 4); - describe("ModelInfoView", () => { let queryClient: QueryClient; @@ -202,10 +195,6 @@ describe("ModelInfoView", () => { mockCredentialCreateCall.mockResolvedValue({}); }); - afterEach(() => { - vi.unstubAllGlobals(); - }); - const wrapper = ({ children }: { children: ReactNode }) => React.createElement(QueryClientProvider, { client: queryClient }, children); @@ -216,70 +205,6 @@ describe("ModelInfoView", () => { }); }); - it("refreshes deprecation banner data after updating model credentials", async () => { - const deprecatedResponse = { - checked_at: "2026-06-01T00:00:00Z", - warn_within_days: 30, - deprecated: [ - { - model_name: "old-model", - litellm_provider: null, - deprecation_date: "2026-06-01", - days_until_deprecation: -1, - status: "deprecated", - successor_model: null, - }, - ], - imminent: [], - upcoming: [], - }; - const currentResponse = { - checked_at: "2026-06-02T00:00:00Z", - warn_within_days: 30, - deprecated: [], - imminent: [ - { - model_name: "current-model", - litellm_provider: null, - deprecation_date: "2026-11-30", - days_until_deprecation: 26, - status: "imminent", - successor_model: null, - }, - ], - upcoming: [], - }; - const fetchMock = vi - .fn() - .mockResolvedValueOnce(Response.json(deprecatedResponse)) - .mockResolvedValueOnce(Response.json(currentResponse)); - vi.stubGlobal("fetch", fetchMock); - - render( - <> - - - , - { wrapper }, - ); - - expect(await screen.findByText("old-model was deprecated on 2026-06-01")).toBeInTheDocument(); - await userEvent.click(screen.getByTestId("update-api-key-button")); - const dialog = screen.getByRole("dialog"); - fireEvent.change(within(dialog).getByPlaceholderText("Enter the new API key"), { - target: { value: "new-api-key" }, - }); - await userEvent.click(within(dialog).getByRole("button", { name: "Update API Key" })); - - expect(await screen.findByText("current-model deprecates on 2026-11-30, in 26 days")).toBeInTheDocument(); - await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); - expect( - fetchMock.mock.calls.map( - ([input]) => new URL(input instanceof Request ? input.url : String(input), "http://localhost").pathname, - ), - ).toEqual(["/model/deprecations", "/model/deprecations"]); - }); - it("shows Kubernetes pods for discovery-enabled models", async () => { mockUseModelsInfo.mockReturnValue({ data: { diff --git a/ui/litellm-dashboard/src/components/model_info_view.tsx b/ui/litellm-dashboard/src/components/model_info_view.tsx index e0498fd1ea9..efa7caf550d 100644 --- a/ui/litellm-dashboard/src/components/model_info_view.tsx +++ b/ui/litellm-dashboard/src/components/model_info_view.tsx @@ -1,6 +1,5 @@ import { useModelCostMap } from "@/app/(dashboard)/hooks/models/useModelCostMap"; import { useModelHub, useModelsInfo } from "@/app/(dashboard)/hooks/models/useModels"; -import { invalidateModelQueries } from "@/app/(dashboard)/hooks/models/invalidateModelQueries"; import { useQueryClient } from "@tanstack/react-query"; import { transformModelData } from "@/app/(dashboard)/models-and-endpoints/utils/modelDataTransformer"; import { KeyIcon, RefreshIcon, TrashIcon } from "@heroicons/react/outline"; @@ -825,7 +824,7 @@ export default function ModelInfoView({ accessToken={accessToken} modelId={modelId} onUpdated={() => { - void invalidateModelQueries(queryClient); + queryClient.invalidateQueries({ queryKey: ["models", "list"] }); }} /> )} diff --git a/ui/litellm-dashboard/src/components/page_metadata.ts b/ui/litellm-dashboard/src/components/page_metadata.ts index 97220a0ef76..80c3a9d2ed7 100644 --- a/ui/litellm-dashboard/src/components/page_metadata.ts +++ b/ui/litellm-dashboard/src/components/page_metadata.ts @@ -20,6 +20,7 @@ export const pageDescriptions: Record = { "tool-policies": "Configure tool use policies and permissions", "vector-stores": "Manage vector databases for embeddings", new_usage: "View usage analytics and metrics", + alerts: "Review and resolve model deprecation alerts across keys and teams", "cost-optimization": "Track and configure cost-saving features: prompt compression, caching, and auto routing", "model-insights": "Model Leaderboard: compare usage, spend, tokens, and task mix across this gateway", logs: "Access request and response logs", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a2073bb00d8..1f12db1c063 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -944,6 +944,44 @@ export interface paths { patch?: never; trace?: never; }; + "/alerting/model-deprecations/state": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Deprecation Alert State */ + get: operations["get_deprecation_alert_state_alerting_model_deprecations_state_get"]; + /** Update Deprecation Alert State */ + put: operations["update_deprecation_alert_state_alerting_model_deprecations_state_put"]; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/alerting/model-deprecations/tags": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Deprecation Usage Tags + * @description Distinct tags observed with this model for a page of affected reporting key IDs. + */ + get: operations["get_deprecation_usage_tags_alerting_model_deprecations_tags_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/alerting/settings": { parameters: { query?: never; @@ -32376,6 +32414,59 @@ export interface components { /** Warnings */ warnings: components["schemas"]["DeploymentWarning"][]; }; + /** DeprecationAlertState */ + DeprecationAlertState: { + /** + * Deprecation Date + * Format: date + */ + deprecation_date: string; + /** Last Seen Date */ + last_seen_date?: string | null; + /** + * Last Seen Requests + * @default 0 + */ + last_seen_requests: number; + /** Model */ + model: string; + /** @default open */ + status: components["schemas"]["DeprecationAlertStatus"]; + /** Updated At */ + updated_at?: string | null; + /** Updated By */ + updated_by?: string | null; + }; + /** + * DeprecationAlertStatus + * @enum {string} + */ + DeprecationAlertStatus: "open" | "resolved" | "dismissed"; + /** DeprecationAlertUpdate */ + DeprecationAlertUpdate: { + /** + * Deprecation Date + * Format: date + */ + deprecation_date: string; + /** + * Last Seen Date + * Format: date + */ + last_seen_date: string; + /** Last Seen Requests */ + last_seen_requests: number; + /** Model */ + model: string; + status: components["schemas"]["DeprecationAlertStatus"]; + }; + /** DeprecationTagUsage */ + DeprecationTagUsage: { + /** Api Key */ + api_key: string; + /** Tag */ + tag: string; + }; /** DiscoverAgentRequest */ DiscoverAgentRequest: { /** @@ -53157,6 +53248,105 @@ export interface operations { }; }; }; + get_deprecation_alert_state_alerting_model_deprecations_state_get: { + parameters: { + query: { + model: string; + deprecation_date: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DeprecationAlertState"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + update_deprecation_alert_state_alerting_model_deprecations_state_put: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["DeprecationAlertUpdate"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DeprecationAlertState"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_deprecation_usage_tags_alerting_model_deprecations_tags_get: { + parameters: { + query: { + model: string; + api_keys: string[]; + start_date: string; + end_date: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["DeprecationTagUsage"][]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; alerting_settings_alerting_settings_get: { parameters: { query?: never;