From 71fde4801605b36858aca006dad16513d9e9cd7f Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Thu, 5 Nov 2026 12:34:17 +0800 Subject: [PATCH] feat(alerts): show recommended model provenance and provider notices (#50162) * feat(alerts): show recommended model provenance and provider notices * fix(ui): avoid nested recommendation source conditionals --- .../proxy/common_utils/model_deprecation.py | 38 +++++++--- litellm/types/proxy/model_deprecation.py | 13 ++++ .../common_utils/test_model_deprecation.py | 71 ++++++++++++++++++- .../ModelDeprecationAlerts/AlertsInbox.tsx | 2 +- .../DeprecationAlertItem.tsx | 2 +- .../MigrationDetail.tsx | 2 +- .../RecommendedReplacement.test.tsx | 47 ++++++++++++ .../RecommendedReplacement.tsx | 51 +++++++++++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 12 ++++ 9 files changed, 219 insertions(+), 19 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/ModelDeprecationAlerts/RecommendedReplacement.test.tsx diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py index 832ebe175b5..8a91c7b0a0c 100644 --- a/litellm/proxy/common_utils/model_deprecation.py +++ b/litellm/proxy/common_utils/model_deprecation.py @@ -16,6 +16,7 @@ from litellm.types.proxy.model_deprecation import ( DeprecationStatus, ModelDeprecationInfo, ModelDeprecationResponse, + SuccessorModelSource, ) if TYPE_CHECKING: @@ -34,6 +35,7 @@ class _ResolvedDeprecation: litellm_provider: str | None copied_from_another_row: bool = False successor_model: str | None = None + deprecation_source: str | None = None def _parse_deprecation_date(raw_value: object) -> date | None: @@ -49,8 +51,8 @@ def _parse_deprecation_date(raw_value: object) -> date | None: return None -def _successor_model(raw_value: object) -> str | None: - """A non-empty successor name, or None when the field is missing, blank, or not a string""" +def _non_empty_string(raw_value: object) -> str | None: + """A non-empty trimmed string, or None for missing, blank, or non-string fields""" return raw_value.strip() or None if isinstance(raw_value, str) else None @@ -72,7 +74,8 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R copied_from_another_row=isinstance(entry, AliasedCostEntry) and "deprecation_date" not in entry.registered_fields and model_key not in catalog, - successor_model=_successor_model(fields.get("successor_model")), + successor_model=_non_empty_string(fields.get("successor_model")), + deprecation_source=_non_empty_string(fields.get("deprecation_source")), ) @@ -156,6 +159,7 @@ def _resolve_deployment_deprecation( litellm_model=model_key, litellm_provider=override_provider if isinstance(override_provider, str) else None, successor_model=from_catalog.successor_model if from_catalog is not None else None, + deprecation_source=from_catalog.deprecation_source if from_catalog is not None else None, ) @@ -174,7 +178,7 @@ def _checked_successor( successor_log_level: int, ) -> str | None: """The deployment's successor_model as set, with a warning when no model in its team scope matches it""" - successor: Final = _successor_model(model_info.get("successor_model")) + successor: Final = _non_empty_string(model_info.get("successor_model")) if successor is None: return None team_id: Final = model_info.get("team_id") @@ -223,6 +227,14 @@ def _build_info( status=_classify(days_until, warn_within_days), litellm_provider=resolved.litellm_provider, successor_model=configured or resolved.successor_model, + successor_model_source=( + SuccessorModelSource.DEPLOYMENT_CONFIG + if configured is not None + else SuccessorModelSource.MODEL_COST_MAP + if resolved.successor_model + else None + ), + deprecation_source=resolved.deprecation_source, ), configured_successor=configured is not None, ) @@ -240,7 +252,7 @@ def _merge_group(group: Sequence[_DeploymentInfo], successor_log_level: int) -> first: Final = group[0].info tiers: Final = tuple( tuple( - deployment.info.successor_model + deployment.info for deployment in group if deployment.configured_successor is configured and deployment.info.successor_model is not None ) @@ -249,15 +261,23 @@ def _merge_group(group: Sequence[_DeploymentInfo], successor_log_level: int) -> successors: Final = next((tier for tier in tiers if tier), ()) if not successors: return first - if len(frozenset(successors)) > 1: + selected: Final = successors[0] + names: Final = frozenset(info.successor_model for info in successors if info.successor_model is not None) + if len(names) > 1: verbose_logger.log( successor_log_level, "model_deprecation: %s names several successors %s, reporting %s", first.model_name, - sorted(frozenset(successors)), - successors[0], + sorted(names), + selected.successor_model, ) - return first.model_copy(update={"successor_model": successors[0]}) + return first.model_copy( + update={ + "successor_model": selected.successor_model, + "successor_model_source": selected.successor_model_source, + "deprecation_source": selected.deprecation_source, + } + ) def _dedupe( diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py index af21cc770de..37c49d737af 100644 --- a/litellm/types/proxy/model_deprecation.py +++ b/litellm/types/proxy/model_deprecation.py @@ -17,6 +17,11 @@ DEPRECATION_IDLE_POLL_SECONDS: Final = 30 DeprecationStatus = Literal["upcoming", "imminent", "deprecated"] +class SuccessorModelSource(str, Enum): + MODEL_COST_MAP = "model_cost_map" + DEPLOYMENT_CONFIG = "deployment_config" + + class ModelDeprecationInfo(LiteLLMBaseModel): model_name: str = Field(description="The public name of the model on the proxy (model_group).") team_public_model_name: str | None = Field( @@ -45,6 +50,14 @@ class ModelDeprecationInfo(LiteLLMBaseModel): ), ) + successor_model_source: SuccessorModelSource | None = Field( + default=None, + description="Whether the recommended successor comes from the model cost map or deployment configuration.", + ) + deprecation_source: str | None = Field( + default=None, description="Provider deprecation notice linked by the selected model cost-map entry." + ) + class ModelDeprecationResponse(LiteLLMBaseModel): deprecated: list[ModelDeprecationInfo] = Field( diff --git a/tests/unit/proxy/common_utils/test_model_deprecation.py b/tests/unit/proxy/common_utils/test_model_deprecation.py index b3c135f1fad..a0fe560e6fb 100644 --- a/tests/unit/proxy/common_utils/test_model_deprecation.py +++ b/tests/unit/proxy/common_utils/test_model_deprecation.py @@ -618,9 +618,7 @@ class TestSuccessorModel: "typo-old": "gpt-nwe", "wildcard-old": "anthropic/x", } - team_deprecation: Final = next( - info for info in snapshot.imminent if info.model_name == "model_name_t1_old" - ) + team_deprecation: Final = next(info for info in snapshot.imminent if info.model_name == "model_name_t1_old") assert team_deprecation.team_public_model_name == "team-old" warnings = [record.getMessage() for record in caplog.records if "names successor" in record.getMessage()] assert len(warnings) == 2 @@ -688,3 +686,70 @@ class TestSuccessorModel: assert "migrate to `<https://evil.example|new>`" in message assert "" not in message assert message.count("migrate to") == 1 + + +class TestRecommendationSources: + def test_should_return_the_cost_map_recommendation_and_provider_notice(self, monkeypatch): + monkeypatch.setattr( + litellm, + "model_cost", + { + "old-model": { + "deprecation_date": "2026-06-10", + "successor_model": "new-model", + "deprecation_source": "https://provider.example/model-retirements", + } + }, + ) + router = _make_router( + [{"model_name": "support", "litellm_params": {"model": "openai/old-model"}, "model_info": {}}] + ) + info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0] + assert info.successor_model == "new-model" + assert info.successor_model_source == "model_cost_map" + assert info.deprecation_source == "https://provider.example/model-retirements" + + def test_should_keep_configured_recommendation_provenance_when_merging_deployments(self, monkeypatch): + monkeypatch.setattr( + litellm, + "model_cost", + {"old-model": {"deprecation_date": "2026-06-10", "successor_model": "catalog-pick"}}, + ) + router = _make_router( + [ + {"model_name": "support", "litellm_params": {"model": "openai/old-model"}, "model_info": {}}, + { + "model_name": "support", + "litellm_params": {"model": "openai/old-model"}, + "model_info": {"successor_model": "configured-pick"}, + }, + ] + ) + info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0] + assert info.successor_model == "configured-pick" + assert info.successor_model_source == "deployment_config" + assert info.deprecation_source is None + + def test_should_carry_the_notice_from_the_same_deployment_as_the_selected_recommendation(self, monkeypatch): + monkeypatch.setattr( + litellm, + "model_cost", + { + "a": {"deprecation_date": "2026-06-10", "deprecation_source": "https://provider.example/a"}, + "b": { + "deprecation_date": "2026-06-10", + "successor_model": "b-next", + "deprecation_source": "https://provider.example/b", + }, + }, + ) + router = _make_router( + [ + {"model_name": "support", "litellm_params": {"model": "openai/a"}, "model_info": {}}, + {"model_name": "support", "litellm_params": {"model": "openai/b"}, "model_info": {}}, + ] + ) + info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0] + assert info.successor_model == "b-next" + assert info.successor_model_source == "model_cost_map" + assert info.deprecation_source == "https://provider.example/b" diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx index 2b3be5904e4..f52af46ca30 100644 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/AlertsInbox.tsx @@ -17,7 +17,7 @@ const columns: ColumnDef[] = [ cell: ({ row }) => (
- +
), }, diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx index dc2d2a848be..277262a0084 100644 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/DeprecationAlertItem.tsx @@ -24,7 +24,7 @@ export default function DeprecationAlertItem({ alert }: { alert: DeprecationAler

{alert.requests.toLocaleString()} requests in the last 30 days · Last seen {alert.lastUsedDate} UTC

- + diff --git a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/MigrationDetail.tsx b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/MigrationDetail.tsx index 0ad842b9078..62fee75071e 100644 --- a/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/MigrationDetail.tsx +++ b/ui/litellm-dashboard/src/components/ModelDeprecationAlerts/MigrationDetail.tsx @@ -203,7 +203,7 @@ export default function MigrationDetail({

Last seen {selected.lastUsedDate} UTC · {selected.requests.toLocaleString()} requests in the last 30 days

- +