mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-18 04:44:37 +00:00
e65e5abc4e
* feat(proxy): name the successor model for deprecating deployments * chore(ui): add successor_model to the generated deprecation api types * fix(proxy): keep the cost map successor when a deployment overrides only the date * fix(proxy): read the cost map successor from metadata.successor The cost map schema rejects a top-level successor_model key, and curated entries already carry the successor under metadata.successor. Read that key, and look it up even when the cost map entry has no deprecation date, so a deployment that sets only its own date still names the successor. * fix(proxy): keep the first deployment when deduping, let a blank successor hide the cost map one, and name a served successor by its model group * fix(proxy): name only the deployment's own successor_model, since cost map successors are LiteLLM ids rather than served model names * fix(proxy): leave out a successor_model the deployment's callers cannot call, with a warning * fix(proxy): report successor_model as the admin set it and only warn when no model matches it --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
536 lines
20 KiB
Python
536 lines
20 KiB
Python
"""Tests for the model deprecation helper module.
|
|
|
|
These tests focus on the helper itself — not on the proxy endpoint or
|
|
Slack integration — so they can run without the full proxy stack.
|
|
"""
|
|
|
|
import logging
|
|
from datetime import date, datetime, timezone
|
|
from unittest.mock import MagicMock
|
|
|
|
|
|
|
|
import litellm
|
|
from litellm import Router
|
|
from litellm._logging import verbose_logger
|
|
from litellm.proxy.common_utils.model_deprecation import (
|
|
_classify,
|
|
_parse_deprecation_date,
|
|
collect_model_deprecations,
|
|
format_deprecation_alert_message,
|
|
)
|
|
|
|
|
|
def _make_router(deployments):
|
|
router = MagicMock()
|
|
router.get_model_list.return_value = deployments
|
|
return router
|
|
|
|
|
|
class TestParseDeprecationDate:
|
|
def test_should_parse_iso_string(self):
|
|
assert _parse_deprecation_date("2026-12-31") == date(2026, 12, 31)
|
|
|
|
def test_should_pass_through_date_object(self):
|
|
d = date(2026, 1, 1)
|
|
assert _parse_deprecation_date(d) == d
|
|
|
|
def test_should_return_none_for_documentation_sentinel(self):
|
|
# The JSON map ships a sentinel string under the "sample_spec" key.
|
|
assert (
|
|
_parse_deprecation_date(
|
|
"date when the model becomes deprecated in the format YYYY-MM-DD"
|
|
)
|
|
is None
|
|
)
|
|
|
|
def test_should_return_none_for_none(self):
|
|
assert _parse_deprecation_date(None) is None
|
|
|
|
def test_should_return_none_for_unsupported_type(self):
|
|
assert _parse_deprecation_date(12345) is None
|
|
|
|
def test_should_narrow_datetime_to_date(self):
|
|
assert _parse_deprecation_date(
|
|
datetime(2026, 12, 31, 23, 59, tzinfo=timezone.utc)
|
|
) == date(2026, 12, 31)
|
|
|
|
|
|
class TestClassify:
|
|
def test_should_classify_past_dates_as_deprecated(self):
|
|
assert _classify(-1, warn_within_days=30) == "deprecated"
|
|
assert _classify(-365, warn_within_days=30) == "deprecated"
|
|
|
|
def test_should_classify_inside_window_as_imminent(self):
|
|
assert _classify(0, warn_within_days=30) == "imminent"
|
|
assert _classify(15, warn_within_days=30) == "imminent"
|
|
assert _classify(30, warn_within_days=30) == "imminent"
|
|
|
|
def test_should_classify_outside_window_as_upcoming(self):
|
|
assert _classify(31, warn_within_days=30) == "upcoming"
|
|
assert _classify(365, warn_within_days=30) == "upcoming"
|
|
|
|
|
|
class TestCollectModelDeprecations:
|
|
def test_should_return_empty_response_when_router_is_none(self):
|
|
snapshot = collect_model_deprecations(llm_router=None)
|
|
assert snapshot.deprecated == []
|
|
assert snapshot.imminent == []
|
|
assert snapshot.upcoming == []
|
|
|
|
def test_should_skip_models_without_deprecation_metadata(self, monkeypatch):
|
|
monkeypatch.setattr(litellm, "model_cost", {})
|
|
router = _make_router(
|
|
[
|
|
{
|
|
"model_name": "gpt-4o",
|
|
"litellm_params": {"model": "openai/gpt-4o"},
|
|
"model_info": {"id": "abc"},
|
|
}
|
|
]
|
|
)
|
|
snapshot = collect_model_deprecations(llm_router=router)
|
|
assert snapshot.deprecated == []
|
|
assert snapshot.imminent == []
|
|
assert snapshot.upcoming == []
|
|
|
|
def test_should_classify_into_three_buckets(self, monkeypatch):
|
|
today = date(2026, 6, 1)
|
|
monkeypatch.setattr(
|
|
litellm,
|
|
"model_cost",
|
|
{
|
|
"deprecated-model": {
|
|
"deprecation_date": "2026-01-01",
|
|
"litellm_provider": "openai",
|
|
},
|
|
"imminent-model": {
|
|
"deprecation_date": "2026-06-15",
|
|
"litellm_provider": "openai",
|
|
},
|
|
"upcoming-model": {
|
|
"deprecation_date": "2027-01-01",
|
|
"litellm_provider": "openai",
|
|
},
|
|
},
|
|
)
|
|
router = _make_router(
|
|
[
|
|
{
|
|
"model_name": "deprecated-alias",
|
|
"litellm_params": {"model": "openai/deprecated-model"},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "imminent-alias",
|
|
"litellm_params": {"model": "imminent-model"},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
{
|
|
"model_name": "upcoming-alias",
|
|
"litellm_params": {"model": "openai/upcoming-model"},
|
|
"model_info": {"id": "3"},
|
|
},
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(
|
|
llm_router=router, warn_within_days=30, today=today
|
|
)
|
|
|
|
assert [m.model_name for m in snapshot.deprecated] == ["deprecated-alias"]
|
|
assert [m.model_name for m in snapshot.imminent] == ["imminent-alias"]
|
|
assert [m.model_name for m in snapshot.upcoming] == ["upcoming-alias"]
|
|
|
|
assert snapshot.deprecated[0].days_until_deprecation < 0
|
|
assert snapshot.imminent[0].days_until_deprecation == 14
|
|
assert snapshot.upcoming[0].days_until_deprecation > 30
|
|
|
|
def test_should_prefer_explicit_deployment_override(self, monkeypatch):
|
|
today = date(2026, 6, 1)
|
|
monkeypatch.setattr(
|
|
litellm,
|
|
"model_cost",
|
|
{"some-model": {"deprecation_date": "2030-01-01"}},
|
|
)
|
|
router = _make_router(
|
|
[
|
|
{
|
|
"model_name": "my-alias",
|
|
"litellm_params": {"model": "some-model"},
|
|
"model_info": {
|
|
"id": "x",
|
|
"deprecation_date": "2026-06-10",
|
|
},
|
|
}
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(
|
|
llm_router=router, warn_within_days=30, today=today
|
|
)
|
|
|
|
assert len(snapshot.imminent) == 1
|
|
assert snapshot.imminent[0].deprecation_date == date(2026, 6, 10)
|
|
|
|
def test_should_dedupe_duplicate_deployments_in_same_group(self, monkeypatch):
|
|
today = date(2026, 6, 1)
|
|
monkeypatch.setattr(
|
|
litellm,
|
|
"model_cost",
|
|
{"shared-model": {"deprecation_date": "2026-06-10"}},
|
|
)
|
|
router = _make_router(
|
|
[
|
|
{
|
|
"model_name": "alias",
|
|
"litellm_params": {"model": "shared-model"},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "alias",
|
|
"litellm_params": {"model": "shared-model"},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(
|
|
llm_router=router, warn_within_days=30, today=today
|
|
)
|
|
|
|
assert len(snapshot.imminent) == 1
|
|
|
|
def test_should_resolve_via_unprefixed_model_name(self, monkeypatch):
|
|
monkeypatch.setattr(
|
|
litellm,
|
|
"model_cost",
|
|
{"gpt-4o": {"deprecation_date": "2026-06-10"}},
|
|
)
|
|
router = _make_router(
|
|
[
|
|
{
|
|
"model_name": "alias",
|
|
"litellm_params": {"model": "openai/gpt-4o"},
|
|
"model_info": {"id": "1"},
|
|
}
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(
|
|
llm_router=router, warn_within_days=30, today=date(2026, 6, 1)
|
|
)
|
|
|
|
assert [m.litellm_model for m in snapshot.imminent] == ["gpt-4o"]
|
|
|
|
def test_should_keep_both_dates_when_group_has_conflicting_dates(self, monkeypatch):
|
|
monkeypatch.setattr(
|
|
litellm,
|
|
"model_cost",
|
|
{"shared-model": {"deprecation_date": "2026-06-10"}},
|
|
)
|
|
router = _make_router(
|
|
[
|
|
{
|
|
"model_name": "alias",
|
|
"litellm_params": {"model": "shared-model"},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "alias",
|
|
"litellm_params": {"model": "shared-model"},
|
|
"model_info": {"id": "2", "deprecation_date": "2027-01-01"},
|
|
},
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(
|
|
llm_router=router, warn_within_days=30, today=date(2026, 6, 1)
|
|
)
|
|
|
|
assert len(snapshot.imminent) == 1
|
|
assert len(snapshot.upcoming) == 1
|
|
|
|
def test_should_resolve_via_base_model(self, monkeypatch):
|
|
today = date(2026, 6, 1)
|
|
monkeypatch.setattr(
|
|
litellm,
|
|
"model_cost",
|
|
{"base-thing": {"deprecation_date": "2026-06-10"}},
|
|
)
|
|
router = _make_router(
|
|
[
|
|
{
|
|
"model_name": "alias",
|
|
"litellm_params": {"model": "azure/some-deployment-name"},
|
|
"model_info": {"id": "1", "base_model": "base-thing"},
|
|
}
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(
|
|
llm_router=router, warn_within_days=30, today=today
|
|
)
|
|
|
|
assert len(snapshot.imminent) == 1
|
|
assert snapshot.imminent[0].litellm_model == "base-thing"
|
|
|
|
|
|
class TestFormatDeprecationAlertMessage:
|
|
def test_should_return_none_when_nothing_to_alert(self):
|
|
snapshot = collect_model_deprecations(llm_router=None)
|
|
assert format_deprecation_alert_message(snapshot) is None
|
|
|
|
def test_should_render_imminent_and_deprecated_sections(self, monkeypatch):
|
|
today = date(2026, 6, 1)
|
|
monkeypatch.setattr(
|
|
litellm,
|
|
"model_cost",
|
|
{
|
|
"dead-model": {
|
|
"deprecation_date": "2026-01-01",
|
|
"litellm_provider": "openai",
|
|
},
|
|
"soon-model": {
|
|
"deprecation_date": "2026-06-15",
|
|
"litellm_provider": "anthropic",
|
|
},
|
|
"later-model": {
|
|
"deprecation_date": "2027-01-01",
|
|
"litellm_provider": "anthropic",
|
|
},
|
|
},
|
|
)
|
|
router = _make_router(
|
|
[
|
|
{
|
|
"model_name": "dead",
|
|
"litellm_params": {"model": "dead-model"},
|
|
"model_info": {"id": "1"},
|
|
},
|
|
{
|
|
"model_name": "soon",
|
|
"litellm_params": {"model": "soon-model"},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
{
|
|
"model_name": "later",
|
|
"litellm_params": {"model": "later-model"},
|
|
"model_info": {"id": "3"},
|
|
},
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(
|
|
llm_router=router, warn_within_days=30, today=today
|
|
)
|
|
message = format_deprecation_alert_message(snapshot)
|
|
|
|
assert message is not None
|
|
assert "Already deprecated" in message
|
|
assert "Deprecating within 30 days" in message
|
|
assert "`dead`" in message
|
|
assert "`soon`" in message
|
|
# Upcoming models must NOT be in the alert (avoid alert fatigue).
|
|
assert "`later`" not in message
|
|
|
|
def test_should_neutralize_slack_markup_from_model_metadata(self):
|
|
today = date(2026, 6, 1)
|
|
router = _make_router(
|
|
[
|
|
{
|
|
"model_name": "<!channel> pwned",
|
|
"litellm_params": {"model": "openai/whatever"},
|
|
"model_info": {
|
|
"id": "1",
|
|
"deprecation_date": "2026-06-10",
|
|
"litellm_provider": "<https://evil.example|openai> & co",
|
|
},
|
|
}
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(
|
|
llm_router=router, warn_within_days=30, today=today
|
|
)
|
|
message = format_deprecation_alert_message(snapshot)
|
|
|
|
assert message is not None
|
|
assert "<!channel>" not in message
|
|
assert "<https://evil.example|openai>" not in message
|
|
assert "<!channel> pwned" in message
|
|
assert "<https://evil.example|openai> & co" in message
|
|
|
|
|
|
class TestSuccessorModel:
|
|
def test_should_surface_the_successor_set_on_the_deployment(self, monkeypatch):
|
|
monkeypatch.setattr(litellm, "model_cost", {})
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "old",
|
|
"litellm_params": {"model": "openai/old-model"},
|
|
"model_info": {"id": "1", "deprecation_date": "2026-06-10", "successor_model": "new"},
|
|
},
|
|
{"model_name": "new", "litellm_params": {"model": "openai/new-model"}, "model_info": {"id": "2"}},
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))
|
|
|
|
assert snapshot.imminent[0].successor_model == "new"
|
|
|
|
def test_should_name_the_deployment_successor_when_the_date_comes_from_the_cost_map(self, monkeypatch):
|
|
monkeypatch.setattr(litellm, "model_cost", {"old-model": {"deprecation_date": "2026-06-10"}})
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "old",
|
|
"litellm_params": {"model": "openai/old-model"},
|
|
"model_info": {"id": "1", "successor_model": "my-pick"},
|
|
},
|
|
{"model_name": "my-pick", "litellm_params": {"model": "openai/pick"}, "model_info": {"id": "2"}},
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))
|
|
|
|
assert snapshot.imminent[0].successor_model == "my-pick"
|
|
|
|
def test_should_ignore_blank_or_non_string_successors(self, monkeypatch):
|
|
monkeypatch.setattr(litellm, "model_cost", {})
|
|
router = _make_router(
|
|
[
|
|
{
|
|
"model_name": "blank",
|
|
"litellm_params": {"model": "openai/a"},
|
|
"model_info": {"id": "1", "deprecation_date": "2026-06-10", "successor_model": " "},
|
|
},
|
|
{
|
|
"model_name": "number",
|
|
"litellm_params": {"model": "openai/b"},
|
|
"model_info": {"id": "2", "deprecation_date": "2026-06-10", "successor_model": 7},
|
|
},
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))
|
|
|
|
assert [m.successor_model for m in snapshot.imminent] == [None, None]
|
|
|
|
def test_should_report_successors_as_set_and_warn_when_no_model_in_the_team_scope_matches(
|
|
self, monkeypatch, caplog
|
|
):
|
|
monkeypatch.setattr(litellm, "model_cost", {})
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "model_name_t1_old",
|
|
"litellm_params": {"model": "openai/team-old"},
|
|
"model_info": {
|
|
"id": "1",
|
|
"team_id": "t1",
|
|
"team_public_model_name": "team-old",
|
|
"deprecation_date": "2026-06-10",
|
|
"successor_model": "team-new",
|
|
},
|
|
},
|
|
{
|
|
"model_name": "model_name_t1_new",
|
|
"litellm_params": {"model": "openai/team-new"},
|
|
"model_info": {"id": "2", "team_id": "t1", "team_public_model_name": "team-new"},
|
|
},
|
|
{
|
|
"model_name": "global-old",
|
|
"litellm_params": {"model": "openai/global-old"},
|
|
"model_info": {"id": "3", "deprecation_date": "2026-06-10", "successor_model": "team-new"},
|
|
},
|
|
{
|
|
"model_name": "typo-old",
|
|
"litellm_params": {"model": "openai/typo-old"},
|
|
"model_info": {"id": "4", "deprecation_date": "2026-06-10", "successor_model": "gpt-nwe"},
|
|
},
|
|
{
|
|
"model_name": "wildcard-old",
|
|
"litellm_params": {"model": "openai/wildcard-old"},
|
|
"model_info": {"id": "5", "deprecation_date": "2026-06-10", "successor_model": "anthropic/x"},
|
|
},
|
|
{"model_name": "anthropic/*", "litellm_params": {"model": "anthropic/*"}, "model_info": {"id": "6"}},
|
|
]
|
|
)
|
|
|
|
with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
|
|
snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))
|
|
|
|
assert {m.model_name: m.successor_model for m in snapshot.imminent} == {
|
|
"model_name_t1_old": "team-new",
|
|
"global-old": "team-new",
|
|
"typo-old": "gpt-nwe",
|
|
"wildcard-old": "anthropic/x",
|
|
}
|
|
warnings = [record.getMessage() for record in caplog.records if "names successor" in record.getMessage()]
|
|
assert len(warnings) == 2
|
|
assert any("global-old names successor team-new" in warning for warning in warnings)
|
|
assert any("typo-old names successor gpt-nwe" in warning for warning in warnings)
|
|
|
|
def test_should_keep_the_first_deployment_and_carry_the_group_successor_when_deduping(self, monkeypatch):
|
|
monkeypatch.setattr(litellm, "model_cost", {})
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "alias",
|
|
"litellm_params": {"model": "openai/a"},
|
|
"model_info": {"id": "1", "deprecation_date": "2026-06-10"},
|
|
},
|
|
{
|
|
"model_name": "alias",
|
|
"litellm_params": {"model": "openai/b"},
|
|
"model_info": {"id": "2", "deprecation_date": "2026-06-10", "successor_model": "alias-v2"},
|
|
},
|
|
{
|
|
"model_name": "alias",
|
|
"litellm_params": {"model": "openai/c"},
|
|
"model_info": {"id": "3", "deprecation_date": "2026-06-10", "successor_model": "alias-v3"},
|
|
},
|
|
{"model_name": "alias-v2", "litellm_params": {"model": "openai/v2"}, "model_info": {"id": "4"}},
|
|
{"model_name": "alias-v3", "litellm_params": {"model": "openai/v3"}, "model_info": {"id": "5"}},
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))
|
|
|
|
assert [(m.litellm_model, m.successor_model) for m in snapshot.imminent] == [("openai/a", "alias-v2")]
|
|
|
|
def test_should_name_the_successor_in_the_slack_entry_and_escape_it(self, monkeypatch):
|
|
monkeypatch.setattr(litellm, "model_cost", {})
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "old",
|
|
"litellm_params": {"model": "openai/old-model"},
|
|
"model_info": {
|
|
"id": "1",
|
|
"deprecation_date": "2026-06-10",
|
|
"successor_model": "<https://evil.example|new>",
|
|
},
|
|
},
|
|
{
|
|
"model_name": "<https://evil.example|new>",
|
|
"litellm_params": {"model": "openai/new-model"},
|
|
"model_info": {"id": "2"},
|
|
},
|
|
{
|
|
"model_name": "orphan",
|
|
"litellm_params": {"model": "openai/orphan-model"},
|
|
"model_info": {"id": "3", "deprecation_date": "2026-06-10"},
|
|
},
|
|
]
|
|
)
|
|
|
|
snapshot = collect_model_deprecations(llm_router=router, warn_within_days=30, today=date(2026, 6, 1))
|
|
message = format_deprecation_alert_message(snapshot)
|
|
|
|
assert message is not None
|
|
assert "migrate to `<https://evil.example|new>`" in message
|
|
assert "<https://evil.example|new>" not in message
|
|
assert message.count("migrate to") == 1
|