mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): prefer a configured successor_model over the cost map's when deployments share a name, and keep the cost map's under a hand-set deprecation_date
This commit is contained in:
parent
42d038036c
commit
dfc238b328
2 changed files with 65 additions and 30 deletions
|
|
@ -130,15 +130,6 @@ def _resolve_deployment_deprecation(
|
|||
litellm_params: Final = _mapping_field(deployment, "litellm_params")
|
||||
model_key: Final = _as_model_key(litellm_params.get("model"))
|
||||
|
||||
override: Final = _parse_deprecation_date(model_info.get("deprecation_date"))
|
||||
if override is not None:
|
||||
override_provider: Final = model_info.get("litellm_provider")
|
||||
return _ResolvedDeprecation(
|
||||
deprecation_date=override,
|
||||
litellm_model=model_key,
|
||||
litellm_provider=override_provider if isinstance(override_provider, str) else None,
|
||||
)
|
||||
|
||||
provider: Final = _deployment_provider(litellm_params)
|
||||
base_model: Final = _as_model_key(model_info.get("base_model"))
|
||||
model_name: Final = model_key.split("/", 1)[1] if model_key is not None and "/" in model_key else model_key
|
||||
|
|
@ -153,7 +144,18 @@ def _resolve_deployment_deprecation(
|
|||
if pick is not None
|
||||
)
|
||||
own_rows: Final = tuple(pick for pick in picks if not pick.copied_from_another_row)
|
||||
return min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None)
|
||||
from_catalog: Final = min(own_rows or picks, key=lambda pick: pick.deprecation_date, default=None)
|
||||
|
||||
override: Final = _parse_deprecation_date(model_info.get("deprecation_date"))
|
||||
if override is None:
|
||||
return from_catalog
|
||||
override_provider: Final = model_info.get("litellm_provider")
|
||||
return _ResolvedDeprecation(
|
||||
deprecation_date=override,
|
||||
litellm_model=model_key,
|
||||
litellm_provider=override_provider if isinstance(override_provider, str) else None,
|
||||
successor_model=from_catalog.successor_model if from_catalog is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def _classify(days_until: int, warn_within_days: int) -> DeprecationStatus:
|
||||
|
|
@ -185,7 +187,7 @@ def _build_info(
|
|||
today: date,
|
||||
warn_within_days: int,
|
||||
catalog: Mapping[str, object],
|
||||
) -> ModelDeprecationInfo | None:
|
||||
) -> _DeploymentInfo | None:
|
||||
model_name: Final = deployment.get("model_name")
|
||||
if not isinstance(model_name, str) or not model_name:
|
||||
return None
|
||||
|
|
@ -195,22 +197,40 @@ def _build_info(
|
|||
return None
|
||||
|
||||
days_until: Final = (resolved.deprecation_date - today).days
|
||||
return ModelDeprecationInfo(
|
||||
model_name=model_name,
|
||||
litellm_model=resolved.litellm_model,
|
||||
deprecation_date=resolved.deprecation_date,
|
||||
days_until_deprecation=days_until,
|
||||
status=_classify(days_until, warn_within_days),
|
||||
litellm_provider=resolved.litellm_provider,
|
||||
successor_model=_checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router)
|
||||
or resolved.successor_model,
|
||||
configured: Final = _checked_successor(model_name, _mapping_field(deployment, "model_info"), llm_router)
|
||||
return _DeploymentInfo(
|
||||
info=ModelDeprecationInfo(
|
||||
model_name=model_name,
|
||||
litellm_model=resolved.litellm_model,
|
||||
deprecation_date=resolved.deprecation_date,
|
||||
days_until_deprecation=days_until,
|
||||
status=_classify(days_until, warn_within_days),
|
||||
litellm_provider=resolved.litellm_provider,
|
||||
successor_model=configured or resolved.successor_model,
|
||||
),
|
||||
configured_successor=configured is not None,
|
||||
)
|
||||
|
||||
|
||||
def _merge_group(group: Sequence[ModelDeprecationInfo]) -> ModelDeprecationInfo:
|
||||
"""Keep the group's first deployment in config order, naming the first successor any of its deployments names"""
|
||||
first: Final = group[0]
|
||||
successors: Final = tuple(model.successor_model for model in group if model.successor_model is not None)
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _DeploymentInfo:
|
||||
info: ModelDeprecationInfo
|
||||
configured_successor: bool
|
||||
|
||||
|
||||
def _merge_group(group: Sequence[_DeploymentInfo]) -> ModelDeprecationInfo:
|
||||
"""Keep the group's first deployment in config order, naming the first successor any of its deployments names,
|
||||
a successor configured in model_info ahead of one the cost map supplies"""
|
||||
first: Final = group[0].info
|
||||
tiers: Final = tuple(
|
||||
tuple(
|
||||
deployment.info.successor_model
|
||||
for deployment in group
|
||||
if deployment.configured_successor is configured and deployment.info.successor_model is not None
|
||||
)
|
||||
for configured in (True, False)
|
||||
)
|
||||
successors: Final = next((tier for tier in tiers if tier), ())
|
||||
if not successors:
|
||||
return first
|
||||
if len(frozenset(successors)) > 1:
|
||||
|
|
@ -224,13 +244,13 @@ def _merge_group(group: Sequence[ModelDeprecationInfo]) -> ModelDeprecationInfo:
|
|||
|
||||
|
||||
def _dedupe(
|
||||
models: Sequence[ModelDeprecationInfo],
|
||||
models: Sequence[_DeploymentInfo],
|
||||
) -> tuple[ModelDeprecationInfo, ...]:
|
||||
"""Report a model group carrying the same date on several deployments once"""
|
||||
ordered: Final = sorted(models, key=lambda model: (model.model_name, model.deprecation_date))
|
||||
ordered: Final = sorted(models, key=lambda model: (model.info.model_name, model.info.deprecation_date))
|
||||
return tuple(
|
||||
_merge_group(tuple(group))
|
||||
for _, group in groupby(ordered, key=lambda model: (model.model_name, model.deprecation_date))
|
||||
for _, group in groupby(ordered, key=lambda model: (model.info.model_name, model.info.deprecation_date))
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -64,9 +64,7 @@ def test_should_bucket_configured_models_by_urgency(authenticated_client):
|
|||
|
||||
|
||||
def test_should_rebucket_with_warn_within_days_override(authenticated_client):
|
||||
response = authenticated_client.get(
|
||||
"/v1/model/deprecations", params={"warn_within_days": 40000}
|
||||
)
|
||||
response = authenticated_client.get("/v1/model/deprecations", params={"warn_within_days": 40000})
|
||||
|
||||
assert response.status_code == 200
|
||||
payload = response.json()
|
||||
|
|
@ -104,6 +102,21 @@ def client_with_cost_map_successors(monkeypatch):
|
|||
"litellm_params": {"model": "retiring-model"},
|
||||
"model_info": {"id": "2", "successor_model": "my-own-pick"},
|
||||
},
|
||||
{
|
||||
"model_name": "shared-name",
|
||||
"litellm_params": {"model": "retiring-model", "api_key": "fake"},
|
||||
"model_info": {"id": "4"},
|
||||
},
|
||||
{
|
||||
"model_name": "shared-name",
|
||||
"litellm_params": {"model": "retiring-model", "api_key": "fake"},
|
||||
"model_info": {"id": "5", "successor_model": "my-own-pick"},
|
||||
},
|
||||
{
|
||||
"model_name": "dated-by-hand",
|
||||
"litellm_params": {"model": "retiring-model", "api_key": "fake"},
|
||||
"model_info": {"id": "6", "deprecation_date": "2020-06-01"},
|
||||
},
|
||||
{
|
||||
"model_name": "orphan-alias",
|
||||
"litellm_params": {"model": "orphan-model"},
|
||||
|
|
@ -123,5 +136,7 @@ def test_cost_map_successor_is_reported_unless_the_deployment_names_its_own(clie
|
|||
assert successors == {
|
||||
"retiring-alias": "fresh-model",
|
||||
"retiring-overridden": "my-own-pick",
|
||||
"shared-name": "my-own-pick",
|
||||
"dated-by-hand": "fresh-model",
|
||||
"orphan-alias": None,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue