mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-05 07:37:50 +00:00
feat(alerts): show recommended model provenance and provider notices (#50162)
* feat(alerts): show recommended model provenance and provider notices * fix(ui): avoid nested recommendation source conditionals
This commit is contained in:
parent
2ffbdf0d7a
commit
71fde48016
9 changed files with 219 additions and 19 deletions
|
|
@ -16,6 +16,7 @@ from litellm.types.proxy.model_deprecation import (
|
|||
DeprecationStatus,
|
||||
ModelDeprecationInfo,
|
||||
ModelDeprecationResponse,
|
||||
SuccessorModelSource,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -34,6 +35,7 @@ class _ResolvedDeprecation:
|
|||
litellm_provider: str | None
|
||||
copied_from_another_row: bool = False
|
||||
successor_model: str | None = None
|
||||
deprecation_source: str | None = None
|
||||
|
||||
|
||||
def _parse_deprecation_date(raw_value: object) -> date | None:
|
||||
|
|
@ -49,8 +51,8 @@ def _parse_deprecation_date(raw_value: object) -> date | None:
|
|||
return None
|
||||
|
||||
|
||||
def _successor_model(raw_value: object) -> str | None:
|
||||
"""A non-empty successor name, or None when the field is missing, blank, or not a string"""
|
||||
def _non_empty_string(raw_value: object) -> str | None:
|
||||
"""A non-empty trimmed string, or None for missing, blank, or non-string fields"""
|
||||
return raw_value.strip() or None if isinstance(raw_value, str) else None
|
||||
|
||||
|
||||
|
|
@ -72,7 +74,8 @@ def _cost_map_lookup(model_key: str | None, catalog: Mapping[str, object]) -> _R
|
|||
copied_from_another_row=isinstance(entry, AliasedCostEntry)
|
||||
and "deprecation_date" not in entry.registered_fields
|
||||
and model_key not in catalog,
|
||||
successor_model=_successor_model(fields.get("successor_model")),
|
||||
successor_model=_non_empty_string(fields.get("successor_model")),
|
||||
deprecation_source=_non_empty_string(fields.get("deprecation_source")),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -156,6 +159,7 @@ def _resolve_deployment_deprecation(
|
|||
litellm_model=model_key,
|
||||
litellm_provider=override_provider if isinstance(override_provider, str) else None,
|
||||
successor_model=from_catalog.successor_model if from_catalog is not None else None,
|
||||
deprecation_source=from_catalog.deprecation_source if from_catalog is not None else None,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -174,7 +178,7 @@ def _checked_successor(
|
|||
successor_log_level: int,
|
||||
) -> str | None:
|
||||
"""The deployment's successor_model as set, with a warning when no model in its team scope matches it"""
|
||||
successor: Final = _successor_model(model_info.get("successor_model"))
|
||||
successor: Final = _non_empty_string(model_info.get("successor_model"))
|
||||
if successor is None:
|
||||
return None
|
||||
team_id: Final = model_info.get("team_id")
|
||||
|
|
@ -223,6 +227,14 @@ def _build_info(
|
|||
status=_classify(days_until, warn_within_days),
|
||||
litellm_provider=resolved.litellm_provider,
|
||||
successor_model=configured or resolved.successor_model,
|
||||
successor_model_source=(
|
||||
SuccessorModelSource.DEPLOYMENT_CONFIG
|
||||
if configured is not None
|
||||
else SuccessorModelSource.MODEL_COST_MAP
|
||||
if resolved.successor_model
|
||||
else None
|
||||
),
|
||||
deprecation_source=resolved.deprecation_source,
|
||||
),
|
||||
configured_successor=configured is not None,
|
||||
)
|
||||
|
|
@ -240,7 +252,7 @@ def _merge_group(group: Sequence[_DeploymentInfo], successor_log_level: int) ->
|
|||
first: Final = group[0].info
|
||||
tiers: Final = tuple(
|
||||
tuple(
|
||||
deployment.info.successor_model
|
||||
deployment.info
|
||||
for deployment in group
|
||||
if deployment.configured_successor is configured and deployment.info.successor_model is not None
|
||||
)
|
||||
|
|
@ -249,15 +261,23 @@ def _merge_group(group: Sequence[_DeploymentInfo], successor_log_level: int) ->
|
|||
successors: Final = next((tier for tier in tiers if tier), ())
|
||||
if not successors:
|
||||
return first
|
||||
if len(frozenset(successors)) > 1:
|
||||
selected: Final = successors[0]
|
||||
names: Final = frozenset(info.successor_model for info in successors if info.successor_model is not None)
|
||||
if len(names) > 1:
|
||||
verbose_logger.log(
|
||||
successor_log_level,
|
||||
"model_deprecation: %s names several successors %s, reporting %s",
|
||||
first.model_name,
|
||||
sorted(frozenset(successors)),
|
||||
successors[0],
|
||||
sorted(names),
|
||||
selected.successor_model,
|
||||
)
|
||||
return first.model_copy(update={"successor_model": successors[0]})
|
||||
return first.model_copy(
|
||||
update={
|
||||
"successor_model": selected.successor_model,
|
||||
"successor_model_source": selected.successor_model_source,
|
||||
"deprecation_source": selected.deprecation_source,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _dedupe(
|
||||
|
|
|
|||
|
|
@ -17,6 +17,11 @@ DEPRECATION_IDLE_POLL_SECONDS: Final = 30
|
|||
DeprecationStatus = Literal["upcoming", "imminent", "deprecated"]
|
||||
|
||||
|
||||
class SuccessorModelSource(str, Enum):
|
||||
MODEL_COST_MAP = "model_cost_map"
|
||||
DEPLOYMENT_CONFIG = "deployment_config"
|
||||
|
||||
|
||||
class ModelDeprecationInfo(LiteLLMBaseModel):
|
||||
model_name: str = Field(description="The public name of the model on the proxy (model_group).")
|
||||
team_public_model_name: str | None = Field(
|
||||
|
|
@ -45,6 +50,14 @@ class ModelDeprecationInfo(LiteLLMBaseModel):
|
|||
),
|
||||
)
|
||||
|
||||
successor_model_source: SuccessorModelSource | None = Field(
|
||||
default=None,
|
||||
description="Whether the recommended successor comes from the model cost map or deployment configuration.",
|
||||
)
|
||||
deprecation_source: str | None = Field(
|
||||
default=None, description="Provider deprecation notice linked by the selected model cost-map entry."
|
||||
)
|
||||
|
||||
|
||||
class ModelDeprecationResponse(LiteLLMBaseModel):
|
||||
deprecated: list[ModelDeprecationInfo] = Field(
|
||||
|
|
|
|||
|
|
@ -618,9 +618,7 @@ class TestSuccessorModel:
|
|||
"typo-old": "gpt-nwe",
|
||||
"wildcard-old": "anthropic/x",
|
||||
}
|
||||
team_deprecation: Final = next(
|
||||
info for info in snapshot.imminent if info.model_name == "model_name_t1_old"
|
||||
)
|
||||
team_deprecation: Final = next(info for info in snapshot.imminent if info.model_name == "model_name_t1_old")
|
||||
assert team_deprecation.team_public_model_name == "team-old"
|
||||
warnings = [record.getMessage() for record in caplog.records if "names successor" in record.getMessage()]
|
||||
assert len(warnings) == 2
|
||||
|
|
@ -688,3 +686,70 @@ class TestSuccessorModel:
|
|||
assert "migrate to `<https://evil.example|new>`" in message
|
||||
assert "<https://evil.example|new>" not in message
|
||||
assert message.count("migrate to") == 1
|
||||
|
||||
|
||||
class TestRecommendationSources:
|
||||
def test_should_return_the_cost_map_recommendation_and_provider_notice(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"model_cost",
|
||||
{
|
||||
"old-model": {
|
||||
"deprecation_date": "2026-06-10",
|
||||
"successor_model": "new-model",
|
||||
"deprecation_source": "https://provider.example/model-retirements",
|
||||
}
|
||||
},
|
||||
)
|
||||
router = _make_router(
|
||||
[{"model_name": "support", "litellm_params": {"model": "openai/old-model"}, "model_info": {}}]
|
||||
)
|
||||
info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0]
|
||||
assert info.successor_model == "new-model"
|
||||
assert info.successor_model_source == "model_cost_map"
|
||||
assert info.deprecation_source == "https://provider.example/model-retirements"
|
||||
|
||||
def test_should_keep_configured_recommendation_provenance_when_merging_deployments(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"model_cost",
|
||||
{"old-model": {"deprecation_date": "2026-06-10", "successor_model": "catalog-pick"}},
|
||||
)
|
||||
router = _make_router(
|
||||
[
|
||||
{"model_name": "support", "litellm_params": {"model": "openai/old-model"}, "model_info": {}},
|
||||
{
|
||||
"model_name": "support",
|
||||
"litellm_params": {"model": "openai/old-model"},
|
||||
"model_info": {"successor_model": "configured-pick"},
|
||||
},
|
||||
]
|
||||
)
|
||||
info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0]
|
||||
assert info.successor_model == "configured-pick"
|
||||
assert info.successor_model_source == "deployment_config"
|
||||
assert info.deprecation_source is None
|
||||
|
||||
def test_should_carry_the_notice_from_the_same_deployment_as_the_selected_recommendation(self, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"model_cost",
|
||||
{
|
||||
"a": {"deprecation_date": "2026-06-10", "deprecation_source": "https://provider.example/a"},
|
||||
"b": {
|
||||
"deprecation_date": "2026-06-10",
|
||||
"successor_model": "b-next",
|
||||
"deprecation_source": "https://provider.example/b",
|
||||
},
|
||||
},
|
||||
)
|
||||
router = _make_router(
|
||||
[
|
||||
{"model_name": "support", "litellm_params": {"model": "openai/a"}, "model_info": {}},
|
||||
{"model_name": "support", "litellm_params": {"model": "openai/b"}, "model_info": {}},
|
||||
]
|
||||
)
|
||||
info = collect_model_deprecations(llm_router=router, today=date(2026, 6, 1)).imminent[0]
|
||||
assert info.successor_model == "b-next"
|
||||
assert info.successor_model_source == "model_cost_map"
|
||||
assert info.deprecation_source == "https://provider.example/b"
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ const columns: ColumnDef<DeprecationAlert>[] = [
|
|||
cell: ({ row }) => (
|
||||
<div className="py-1">
|
||||
<DeprecationModel alert={row.original} />
|
||||
<RecommendedReplacement model={row.original.model.successor_model} />
|
||||
<RecommendedReplacement recommendation={row.original.model} />
|
||||
</div>
|
||||
),
|
||||
},
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ export default function DeprecationAlertItem({ alert }: { alert: DeprecationAler
|
|||
<p className="mt-1 text-xs text-muted-foreground">
|
||||
{alert.requests.toLocaleString()} requests in the last 30 days · Last seen {alert.lastUsedDate} UTC
|
||||
</p>
|
||||
<RecommendedReplacement model={alert.model.successor_model} />
|
||||
<RecommendedReplacement recommendation={alert.model} />
|
||||
</div>
|
||||
</div>
|
||||
<a className={cn(buttonVariants({ size: "sm" }), "self-start")} href={migrationHref(alert.usageModel)}>
|
||||
|
|
|
|||
|
|
@ -203,7 +203,7 @@ export default function MigrationDetail({
|
|||
<p className="mt-1 text-sm text-muted-foreground">
|
||||
Last seen {selected.lastUsedDate} UTC · {selected.requests.toLocaleString()} requests in the last 30 days
|
||||
</p>
|
||||
<RecommendedReplacement model={selected.model.successor_model} />
|
||||
<RecommendedReplacement recommendation={selected.model} />
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Button variant="outline" onClick={copyFollowUp}>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
import { render, screen } from "@testing-library/react";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import RecommendedReplacement from "./RecommendedReplacement";
|
||||
|
||||
const recommendation = {
|
||||
successor_model: "new-model",
|
||||
successor_model_source: "model_cost_map" as const,
|
||||
deprecation_source: "https://provider.example/model-retirements",
|
||||
};
|
||||
|
||||
describe("recommended replacement", () => {
|
||||
it("should render the model with its cost-map source and provider notice", () => {
|
||||
render(<RecommendedReplacement recommendation={recommendation} />);
|
||||
expect(screen.getByText("new-model")).toBeInTheDocument();
|
||||
expect(screen.getByRole("link", { name: "LiteLLM model cost map" })).toHaveAttribute(
|
||||
"href",
|
||||
"https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json",
|
||||
);
|
||||
expect(screen.getByRole("link", { name: /Provider deprecation notice/ })).toHaveAttribute(
|
||||
"href",
|
||||
recommendation.deprecation_source,
|
||||
);
|
||||
});
|
||||
it("should identify deployment overrides without attributing them to the cost map", () => {
|
||||
render(
|
||||
<RecommendedReplacement recommendation={{ ...recommendation, successor_model_source: "deployment_config" }} />,
|
||||
);
|
||||
expect(screen.getByText(/Deployment configuration/)).toBeInTheDocument();
|
||||
expect(screen.queryByRole("link", { name: "LiteLLM model cost map" })).not.toBeInTheDocument();
|
||||
});
|
||||
it("should omit provenance when no replacement is provided", () => {
|
||||
render(<RecommendedReplacement recommendation={{}} />);
|
||||
expect(screen.getByText(/Not provided/)).toBeInTheDocument();
|
||||
expect(screen.queryByText(/Source:/)).not.toBeInTheDocument();
|
||||
});
|
||||
it("should avoid inventing a source when an older proxy only returns a model name", () => {
|
||||
render(<RecommendedReplacement recommendation={{ successor_model: "new-model" }} />);
|
||||
expect(screen.getByText(/Source: Not provided/)).toBeInTheDocument();
|
||||
expect(screen.queryByRole("link")).not.toBeInTheDocument();
|
||||
});
|
||||
it("should omit provider links with an unsafe URL scheme", () => {
|
||||
render(
|
||||
<RecommendedReplacement recommendation={{ ...recommendation, deprecation_source: "javascript:alert(1)" }} />,
|
||||
);
|
||||
expect(screen.queryByRole("link", { name: /Provider deprecation notice/ })).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,7 +1,50 @@
|
|||
export default function RecommendedReplacement({ model }: { model: string | null | undefined }) {
|
||||
import type { components } from "@/lib/http/schema";
|
||||
|
||||
type Recommendation = Pick<
|
||||
components["schemas"]["ModelDeprecationInfo"],
|
||||
"successor_model" | "successor_model_source" | "deprecation_source"
|
||||
>;
|
||||
|
||||
const SOURCE_LABELS = {
|
||||
model_cost_map: "LiteLLM model cost map",
|
||||
deployment_config: "Deployment configuration",
|
||||
} as const;
|
||||
|
||||
export default function RecommendedReplacement({ recommendation }: { recommendation: Recommendation }) {
|
||||
const { successor_model: model, successor_model_source: source, deprecation_source: notice } = recommendation;
|
||||
const sourceLabel = (source && SOURCE_LABELS[source]) || "Not provided";
|
||||
const noticeUrl = notice && /^https?:\/\//i.test(notice) ? notice : null;
|
||||
|
||||
return (
|
||||
<p className="mt-2 text-xs text-muted-foreground">
|
||||
Recommended new model: <span className="font-medium text-foreground">{model || "Not provided"}</span>
|
||||
</p>
|
||||
<div className="mt-2 space-y-1 text-xs text-muted-foreground">
|
||||
<p>
|
||||
Recommended new model:{" "}
|
||||
{model ? <code className="rounded bg-muted px-1.5 py-0.5 text-foreground">{model}</code> : "Not provided"}
|
||||
</p>
|
||||
{model && (
|
||||
<p className="flex flex-wrap items-center gap-x-2 gap-y-1">
|
||||
<span>
|
||||
Source:{" "}
|
||||
{source === "model_cost_map" ? (
|
||||
<a
|
||||
href="https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="underline underline-offset-2"
|
||||
>
|
||||
{sourceLabel}
|
||||
</a>
|
||||
) : (
|
||||
sourceLabel
|
||||
)}
|
||||
</span>
|
||||
{noticeUrl && (
|
||||
<a href={noticeUrl} target="_blank" rel="noopener noreferrer" className="underline underline-offset-2">
|
||||
Provider deprecation notice ↗
|
||||
</a>
|
||||
)}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
12
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
12
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -39114,6 +39114,11 @@ export interface components {
|
|||
* @description The date (UTC) when the model becomes deprecated.
|
||||
*/
|
||||
deprecation_date: string;
|
||||
/**
|
||||
* Deprecation Source
|
||||
* @description Provider deprecation notice linked by the selected model cost-map entry.
|
||||
*/
|
||||
deprecation_source?: string | null;
|
||||
/**
|
||||
* Litellm Model
|
||||
* @description The underlying litellm model string the deprecation date is sourced from.
|
||||
|
|
@ -39140,6 +39145,8 @@ export interface components {
|
|||
* @description The model to migrate to: model_info.successor_model on the deployment when set, else the successor_model the cost map names for the deprecated model.
|
||||
*/
|
||||
successor_model?: string | null;
|
||||
/** @description Whether the recommended successor comes from the model cost map or deployment configuration. */
|
||||
successor_model_source?: components["schemas"]["SuccessorModelSource"] | null;
|
||||
/**
|
||||
* Team Public Model Name
|
||||
* @description The model name visible to the team, when configured.
|
||||
|
|
@ -46923,6 +46930,11 @@ export interface components {
|
|||
[key: string]: unknown;
|
||||
};
|
||||
};
|
||||
/**
|
||||
* SuccessorModelSource
|
||||
* @enum {string}
|
||||
*/
|
||||
SuccessorModelSource: "model_cost_map" | "deployment_config";
|
||||
/** SuggestTemplatesRequest */
|
||||
SuggestTemplatesRequest: {
|
||||
/** Attack Examples */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue