mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(aiand): share the UTC retirement check with Together AI and the health check sort
This commit is contained in:
parent
95ad02f344
commit
2ef37fee94
6 changed files with 57 additions and 24 deletions
|
|
@ -549,7 +549,7 @@ _key_management_system: Optional["KeyManagementSystem"] = None
|
|||
output_parse_pii: bool = False
|
||||
#############################################
|
||||
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, mark_litellm_import_complete
|
||||
from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date as _is_past_deprecation_date
|
||||
from litellm.litellm_core_utils.model_retirement import is_hidden_as_retired
|
||||
|
||||
model_cost = get_model_cost_map(url=model_cost_map_url)
|
||||
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
|
||||
|
|
@ -863,12 +863,12 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
if "-to-" not in key and "fireworks-ai-default" not in key:
|
||||
fireworks_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "together_ai":
|
||||
if _is_past_deprecation_date(value):
|
||||
if is_hidden_as_retired(value):
|
||||
together_ai_models.discard(key)
|
||||
elif not key.startswith("together-ai-"):
|
||||
together_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "baseten":
|
||||
if _is_past_deprecation_date(value):
|
||||
if is_hidden_as_retired(value):
|
||||
baseten_models.discard(key)
|
||||
else:
|
||||
baseten_models.add(key)
|
||||
|
|
@ -1041,7 +1041,7 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
elif value.get("litellm_provider") == "coralbricks":
|
||||
coralbricks_models.add(key)
|
||||
elif value.get("litellm_provider") == "aiand":
|
||||
if _is_past_deprecation_date(value):
|
||||
if is_hidden_as_retired(value):
|
||||
aiand_models.discard(key)
|
||||
else:
|
||||
aiand_models.add(key)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import datetime
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, cast
|
||||
|
|
@ -9,6 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
|
|||
import litellm
|
||||
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
|
||||
from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date
|
||||
|
||||
|
||||
def _form_field_value(value: object) -> str:
|
||||
|
|
@ -189,18 +189,11 @@ _COST_MAP_DEPRECATION: Final = TypeAdapter(_CostMapDeprecation)
|
|||
|
||||
|
||||
def _is_past_deprecation_date(cost_map_key: str) -> bool:
|
||||
deprecation_date: Final = _COST_MAP_DEPRECATION.validate_python(
|
||||
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
|
||||
).get("deprecation_date")
|
||||
if not isinstance(deprecation_date, str):
|
||||
return False
|
||||
try:
|
||||
return (
|
||||
datetime.date.fromisoformat(deprecation_date)
|
||||
<= datetime.datetime.now(datetime.timezone.utc).astimezone().date()
|
||||
return is_past_deprecation_date(
|
||||
_COST_MAP_DEPRECATION.validate_python(
|
||||
litellm.model_cost.get(cost_map_key) or {} # pyright: ignore[reportUnknownMemberType] # untyped cost map JSON
|
||||
)
|
||||
except ValueError:
|
||||
return False
|
||||
)
|
||||
|
||||
|
||||
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float, bool]]:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import datetime
|
|||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
PROVIDERS_HIDING_RETIRED_MODELS: Final = frozenset({"aiand", "baseten"})
|
||||
PROVIDERS_HIDING_RETIRED_MODELS: Final = frozenset({"aiand", "baseten", "together_ai"})
|
||||
|
||||
|
||||
def is_past_deprecation_date(value: Mapping[str, object]) -> bool:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import datetime
|
||||
import time
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -245,3 +247,39 @@ def test_rank_chat_models_flags_a_retired_model_the_provider_lists_without_its_c
|
|||
(ranked_model.model, ranked_model.is_past_deprecation_date)
|
||||
for ranked_model in rank_chat_models_from_llm_provider("vertex_ai")
|
||||
] == [("fake-retired-vertex-chat", True), ("fake-live-vertex-chat", False)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("host_zone", ["Etc/GMT+12", "Etc/GMT-14"])
|
||||
def test_rank_chat_models_flags_retirement_by_the_utc_date_whatever_the_host_zone(
|
||||
monkeypatch: pytest.MonkeyPatch, host_zone: str
|
||||
) -> None:
|
||||
utc_today: Final = datetime.datetime.now(datetime.timezone.utc).date()
|
||||
retirement_dates: Final = {
|
||||
"deepseek/fake-retires-today-chat": utc_today,
|
||||
"deepseek/fake-retires-tomorrow-chat": utc_today + datetime.timedelta(days=1),
|
||||
}
|
||||
for name, retirement_date in retirement_dates.items():
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
name,
|
||||
{
|
||||
"input_cost_per_token": 1e-12,
|
||||
"output_cost_per_token": 1e-12,
|
||||
"mode": "chat",
|
||||
"litellm_provider": PICKER_PROVIDER,
|
||||
"deprecation_date": retirement_date.isoformat(),
|
||||
},
|
||||
)
|
||||
monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(retirement_dates))
|
||||
monkeypatch.setenv("TZ", host_zone)
|
||||
time.tzset()
|
||||
try:
|
||||
flags: Final = {
|
||||
ranked_model.model: ranked_model.is_past_deprecation_date
|
||||
for ranked_model in rank_chat_models_from_llm_provider(PICKER_PROVIDER)
|
||||
}
|
||||
finally:
|
||||
monkeypatch.undo()
|
||||
time.tzset()
|
||||
|
||||
assert flags == {"deepseek/fake-retires-today-chat": True, "deepseek/fake-retires-tomorrow-chat": False}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import pytest
|
|||
import respx
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date
|
||||
from litellm.llms.baseten.chat import BasetenConfig
|
||||
from litellm.utils import _model_cache, get_valid_models
|
||||
|
||||
|
|
@ -65,7 +66,7 @@ class TestBasetenModelDiscovery:
|
|||
servable_registry_models: Final = {
|
||||
key
|
||||
for key, value in litellm.model_cost.items()
|
||||
if value.get("litellm_provider") == "baseten" and not litellm._is_past_deprecation_date(value)
|
||||
if value.get("litellm_provider") == "baseten" and not is_past_deprecation_date(value)
|
||||
}
|
||||
|
||||
assert servable_registry_models
|
||||
|
|
@ -116,7 +117,7 @@ class TestBasetenModelDiscovery:
|
|||
],
|
||||
)
|
||||
def test_is_past_deprecation_date(self, value, expected):
|
||||
assert litellm._is_past_deprecation_date(value) is expected
|
||||
assert is_past_deprecation_date(value) is expected
|
||||
|
||||
def test_get_models_calls_model_api(self, respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.get("https://inference.baseten.co/v1/models").mock(
|
||||
|
|
@ -172,7 +173,7 @@ class TestBasetenModelDiscovery:
|
|||
|
||||
@patch.dict(os.environ, {"BASETEN_API_KEY": "test-key"})
|
||||
def test_live_catalog_keeps_a_model_the_cost_map_dates_as_retired(self, respx_mock: respx.MockRouter) -> None:
|
||||
assert litellm._is_past_deprecation_date(litellm.model_cost["baseten/zai-org/GLM-5"])
|
||||
assert is_past_deprecation_date(litellm.model_cost["baseten/zai-org/GLM-5"])
|
||||
_model_cache.flush_cache()
|
||||
respx_mock.get("https://inference.baseten.co/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "zai-org/GLM-5"}]})
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ import respx
|
|||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.llm_request_utils import pick_cheapest_chat_models_from_llm_provider
|
||||
from litellm.litellm_core_utils.health_check_helpers import wildcard_health_check_models
|
||||
from litellm.litellm_core_utils.model_retirement import is_past_deprecation_date
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
|
@ -128,7 +129,7 @@ def test_aiand_models_by_provider_leaves_out_models_retired_at_load():
|
|||
retired: Final = frozenset(
|
||||
name
|
||||
for name, row in litellm.model_cost.items()
|
||||
if row.get("litellm_provider") == "aiand" and litellm._is_past_deprecation_date(row)
|
||||
if row.get("litellm_provider") == "aiand" and is_past_deprecation_date(row)
|
||||
)
|
||||
|
||||
assert retired
|
||||
|
|
@ -149,11 +150,11 @@ def test_aiand_wildcard_health_check_skips_a_model_whose_retirement_date_passes_
|
|||
monkeypatch.setitem(litellm.model_cost, model, live_row)
|
||||
try:
|
||||
litellm.add_known_models({model: live_row})
|
||||
assert model in pick_cheapest_chat_models_from_llm_provider("aiand", n=1)
|
||||
assert model in wildcard_health_check_models(wildcard="aiand/*", custom_llm_provider="aiand")
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost, model, {**live_row, "deprecation_date": "2000-01-01"})
|
||||
|
||||
assert model not in pick_cheapest_chat_models_from_llm_provider("aiand", n=3)
|
||||
assert model not in wildcard_health_check_models(wildcard="aiand/*", custom_llm_provider="aiand")
|
||||
finally:
|
||||
litellm.aiand_models.discard(model)
|
||||
litellm.add_known_models({})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue