feat(alerts): show recommended model provenance and provider notices (#50162)

* feat(alerts): show recommended model provenance and provider notices

* fix(ui): avoid nested recommendation source conditionals
This commit is contained in:
ishaan-j-berri 2026-11-05 12:34:17 +08:00 • committed by GitHub
parent 2ffbdf0d7a
commit 71fde48016
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 219 additions and 19 deletions

View file

@ -16,6 +16,7 @@ from litellm.types.proxy.model_deprecation import (
DeprecationStatus,
ModelDeprecationInfo,
ModelDeprecationResponse,
SuccessorModelSource,
)
if TYPE_CHECKING:
@ -34,6 +35,7 @@ class _ResolvedDeprecation:
litellm_provider: str | None
copied_from_another_row: bool = False
successor_model: str | None = None
deprecation_source: str | None = None
def _parse_deprecation_date(raw_value: object) -> date | None:
@ -49,8 +51,8 @@ def _parse_deprecation_date(raw_value: object) -> date | None:
return None
def _successor_model(raw_value: object) -> str | None:
"""A non-empty successor name, or None when the field is missing, blank, or not a string"""
def _non_empty_string(raw_value: object) -> str | None:
"""A non-empty trimmed string, or None for missing, blank, or non-string fields"""
return raw_value.strip() or None if isinstance(raw_value, str) else None
@ -72,7 +74,8 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R
copied_from_another_row=isinstance(entry, AliasedCostEntry)
and "deprecation_date" not in entry.registered_fields
and model_key not in catalog,
successor_model=_successor_model(fields.get("successor_model")),
successor_model=_non_empty_string(fields.get("successor_model")),
deprecation_source=_non_empty_string(fields.get("deprecation_source")),
)
@ -156,6 +159,7 @@ def _resolve_deployment_deprecation(
litellm_model=model_key,
litellm_provider=override_provider if isinstance(override_provider, str) else None,
successor_model=from_catalog.successor_model if from_catalog is not None else None,
deprecation_source=from_catalog.deprecation_source if from_catalog is not None else None,
)
@ -174,7 +178,7 @@ def _checked_successor(
successor_log_level: int,
) -> str | None:
"""The deployment's successor_model as set, with a warning when no model in its team scope matches it"""
successor: Final = _successor_model(model_info.get("successor_model"))
successor: Final = _non_empty_string(model_info.get("successor_model"))
if successor is None:
return None
team_id: Final = model_info.get("team_id")
@ -223,6 +227,14 @@ def _build_info(
status=_classify(days_until, warn_within_days),
litellm_provider=resolved.litellm_provider,
successor_model=configured or resolved.successor_model,
successor_model_source=(
SuccessorModelSource.DEPLOYMENT_CONFIG
if configured is not None
else SuccessorModelSource.MODEL_COST_MAP
if resolved.successor_model
else None
),
deprecation_source=resolved.deprecation_source,
),
configured_successor=configured is not None,
)
@ -240,7 +252,7 @@ def _merge_group(group: Sequence[_DeploymentInfo], successor_log_level: int) ->
first: Final = group[0].info
tiers: Final = tuple(
tuple(
deployment.info.successor_model
deployment.info
for deployment in group
if deployment.configured_successor is configured and deployment.info.successor_model is not None
)
@ -249,15 +261,23 @@ def _merge_group(group: Sequence[_DeploymentInfo], successor_log_level: int) ->
successors: Final = next((tier for tier in tiers if tier), ())
if not successors:
return first
if len(frozenset(successors)) > 1:
selected: Final = successors[0]
names: Final = frozenset(info.successor_model for info in successors if info.successor_model is not None)
if len(names) > 1:
verbose_logger.log(
successor_log_level,
"model_deprecation: %s names several successors %s, reporting %s",
first.model_name,
sorted(frozenset(successors)),
successors[0],
sorted(names),
selected.successor_model,
)
return first.model_copy(update={"successor_model": successors[0]})
return first.model_copy(
update={
"successor_model": selected.successor_model,
"successor_model_source": selected.successor_model_source,
"deprecation_source": selected.deprecation_source,
}
)
def _dedupe(

View file

@ -17,6 +17,11 @@ DEPRECATION_IDLE_POLL_SECONDS: Final = 30
DeprecationStatus = Literal["upcoming", "imminent", "deprecated"]
class SuccessorModelSource(str, Enum):
MODEL_COST_MAP = "model_cost_map"
DEPLOYMENT_CONFIG = "deployment_config"
class ModelDeprecationInfo(LiteLLMBaseModel):
model_name: str = Field(description="The public name of the model on the proxy (model_group).")
team_public_model_name: str | None = Field(
@ -45,6 +50,14 @@ class ModelDeprecationInfo(LiteLLMBaseModel):
),
)
successor_model_source: SuccessorModelSource | None = Field(
default=None,
description="Whether the recommended successor comes from the model cost map or deployment configuration.",
)
deprecation_source: str | None = Field(
default=None, description="Provider deprecation notice linked by the selected model cost-map entry."
)
class ModelDeprecationResponse(LiteLLMBaseModel):
deprecated: list[ModelDeprecationInfo] = Field(

View file

@ -618,9 +618,7 @@ class TestSuccessorModel:
"typo-old": "gpt-nwe",
"wildcard-old": "anthropic/x",
}
team_deprecation: Final = next(
info for info in snapshot.imminent if info.model_name == "model_name_t1_old"
)
team_deprecation: Final = next(info for info in snapshot.imminent if info.model_name == "model_name_t1_old")
assert team_deprecation.team_public_model_name == "team-old"
warnings = [record.getMessage() for record in caplog.records if "names successor" in record.getMessage()]
assert len(warnings) == 2
@ -688,3 +686,70 @@ class TestSuccessorModel:
assert "migrate to `<https://evil.example|new>`" in message
assert "<https://evil.example|new>" not in message
assert message.count("migrate to") == 1
class TestRecommendationSources:
def test_should_return_the_cost_map_recommendation_and_provider_notice(self, monkeypatch):
monkeypatch.setattr(
litellm,
"model_cost",
{
"old-model": {
"deprecation_date": "2026-06-10",
"successor_model": "new-model",
"deprecation_source": "https://provider.example/model-retirements",
}
},
)
router = _make_router(
[{"model_name": "support", "litellm_params": {"model": "openai/old-model"}, "model_info": {}}]
)
info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0]
assert info.successor_model == "new-model"
assert info.successor_model_source == "model_cost_map"
assert info.deprecation_source == "https://provider.example/model-retirements"
def test_should_keep_configured_recommendation_provenance_when_merging_deployments(self, monkeypatch):
monkeypatch.setattr(
litellm,
"model_cost",
{"old-model": {"deprecation_date": "2026-06-10", "successor_model": "catalog-pick"}},
)
router = _make_router(
[
{"model_name": "support", "litellm_params": {"model": "openai/old-model"}, "model_info": {}},
{
"model_name": "support",
"litellm_params": {"model": "openai/old-model"},
"model_info": {"successor_model": "configured-pick"},
},
]
)
info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0]
assert info.successor_model == "configured-pick"
assert info.successor_model_source == "deployment_config"
assert info.deprecation_source is None
def test_should_carry_the_notice_from_the_same_deployment_as_the_selected_recommendation(self, monkeypatch):
monkeypatch.setattr(
litellm,
"model_cost",
{
"a": {"deprecation_date": "2026-06-10", "deprecation_source": "https://provider.example/a"},
"b": {
"deprecation_date": "2026-06-10",
"successor_model": "b-next",
"deprecation_source": "https://provider.example/b",
},
},
)
router = _make_router(
[
{"model_name": "support", "litellm_params": {"model": "openai/a"}, "model_info": {}},
{"model_name": "support", "litellm_params": {"model": "openai/b"}, "model_info": {}},
]
)
info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0]
assert info.successor_model == "b-next"
assert info.successor_model_source == "model_cost_map"
assert info.deprecation_source == "https://provider.example/b"

View file

@ -17,7 +17,7 @@ const columns: ColumnDef<DeprecationAlert>[] = [
cell: ({ row }) => (
<div className="py-1">
<DeprecationModel alert={row.original} />
<RecommendedReplacement model={row.original.model.successor_model} />
<RecommendedReplacement recommendation={row.original.model} />
</div>
),
},

View file

@ -24,7 +24,7 @@ export default function DeprecationAlertItem({ alert }: { alert: DeprecationAler
<p className="mt-1 text-xs text-muted-foreground">
{alert.requests.toLocaleString()} requests in the last 30 days · Last seen {alert.lastUsedDate} UTC
</p>
<RecommendedReplacement model={alert.model.successor_model} />
<RecommendedReplacement recommendation={alert.model} />
</div>
</div>
<a className={cn(buttonVariants({ size: "sm" }), "self-start")} href={migrationHref(alert.usageModel)}>

View file

@ -203,7 +203,7 @@ export default function MigrationDetail({
<p className="mt-1 text-sm text-muted-foreground">
Last seen {selected.lastUsedDate} UTC · {selected.requests.toLocaleString()} requests in the last 30 days
</p>
<RecommendedReplacement model={selected.model.successor_model} />
<RecommendedReplacement recommendation={selected.model} />
</div>
<div className="flex flex-wrap gap-2">
<Button variant="outline" onClick={copyFollowUp}>

View file

@ -0,0 +1,47 @@
import { render, screen } from "@testing-library/react";
import { describe, expect, it } from "vitest";
import RecommendedReplacement from "./RecommendedReplacement";
const recommendation = {
successor_model: "new-model",
successor_model_source: "model_cost_map" as const,
deprecation_source: "https://provider.example/model-retirements",
};
describe("recommended replacement", () => {
it("should render the model with its cost-map source and provider notice", () => {
render(<RecommendedReplacement recommendation={recommendation} />);
expect(screen.getByText("new-model")).toBeInTheDocument();
expect(screen.getByRole("link", { name: "LiteLLM model cost map" })).toHaveAttribute(
"href",
"https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json",
);
expect(screen.getByRole("link", { name: /Provider deprecation notice/ })).toHaveAttribute(
"href",
recommendation.deprecation_source,
);
});
it("should identify deployment overrides without attributing them to the cost map", () => {
render(
<RecommendedReplacement recommendation={{ ...recommendation, successor_model_source: "deployment_config" }} />,
);
expect(screen.getByText(/Deployment configuration/)).toBeInTheDocument();
expect(screen.queryByRole("link", { name: "LiteLLM model cost map" })).not.toBeInTheDocument();
});
it("should omit provenance when no replacement is provided", () => {
render(<RecommendedReplacement recommendation={{}} />);
expect(screen.getByText(/Not provided/)).toBeInTheDocument();
expect(screen.queryByText(/Source:/)).not.toBeInTheDocument();
});
it("should avoid inventing a source when an older proxy only returns a model name", () => {
render(<RecommendedReplacement recommendation={{ successor_model: "new-model" }} />);
expect(screen.getByText(/Source: Not provided/)).toBeInTheDocument();
expect(screen.queryByRole("link")).not.toBeInTheDocument();
});
it("should omit provider links with an unsafe URL scheme", () => {
render(
<RecommendedReplacement recommendation={{ ...recommendation, deprecation_source: "javascript:alert(1)" }} />,
);
expect(screen.queryByRole("link", { name: /Provider deprecation notice/ })).not.toBeInTheDocument();
});
});

View file

@ -1,7 +1,50 @@
export default function RecommendedReplacement({ model }: { model: string | null | undefined }) {
import type { components } from "@/lib/http/schema";
type Recommendation = Pick<
components["schemas"]["ModelDeprecationInfo"],
"successor_model" | "successor_model_source" | "deprecation_source"
>;
const SOURCE_LABELS = {
model_cost_map: "LiteLLM model cost map",
deployment_config: "Deployment configuration",
} as const;
export default function RecommendedReplacement({ recommendation }: { recommendation: Recommendation }) {
const { successor_model: model, successor_model_source: source, deprecation_source: notice } = recommendation;
const sourceLabel = (source && SOURCE_LABELS[source]) || "Not provided";
const noticeUrl = notice && /^https?:\/\//i.test(notice) ? notice : null;
return (
<p className="mt-2 text-xs text-muted-foreground">
Recommended new model: <span className="font-medium text-foreground">{model || "Not provided"}</span>
</p>
<div className="mt-2 space-y-1 text-xs text-muted-foreground">
<p>
Recommended new model:{" "}
{model ? <code className="rounded bg-muted px-1.5 py-0.5 text-foreground">{model}</code> : "Not provided"}
</p>
{model && (
<p className="flex flex-wrap items-center gap-x-2 gap-y-1">
<span>
Source:{" "}
{source === "model_cost_map" ? (
<a
href="https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
target="_blank"
rel="noopener noreferrer"
className="underline underline-offset-2"
>
{sourceLabel}
</a>
) : (
sourceLabel
)}
</span>
{noticeUrl && (
<a href={noticeUrl} target="_blank" rel="noopener noreferrer" className="underline underline-offset-2">
Provider deprecation notice ↗
</a>
)}
</p>
)}
</div>
);
}

View file

@ -39114,6 +39114,11 @@ export interface components {
* @description The date (UTC) when the model becomes deprecated.
*/
deprecation_date: string;
/**
* Deprecation Source
* @description Provider deprecation notice linked by the selected model cost-map entry.
*/
deprecation_source?: string | null;
/**
* Litellm Model
* @description The underlying litellm model string the deprecation date is sourced from.
@ -39140,6 +39145,8 @@ export interface components {
* @description The model to migrate to: model_info.successor_model on the deployment when set, else the successor_model the cost map names for the deprecated model.
*/
successor_model?: string | null;
/** @description Whether the recommended successor comes from the model cost map or deployment configuration. */
successor_model_source?: components["schemas"]["SuccessorModelSource"] | null;
/**
* Team Public Model Name
* @description The model name visible to the team, when configured.
@ -46923,6 +46930,11 @@ export interface components {
[key: string]: unknown;
};
};
/**
* SuccessorModelSource
* @enum {string}
*/
SuccessorModelSource: "model_cost_map" | "deployment_config";
/** SuggestTemplatesRequest */
SuggestTemplatesRequest: {
/** Attack Examples */