mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(aiand): hide retired models only from bundled listings, including the wildcard health check
This commit is contained in:
parent
c19fcf9d5f
commit
d41cd3531b
4 changed files with 74 additions and 6 deletions
|
|
@ -151,10 +151,12 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
|
|||
Returns:
|
||||
list[str]: A list of the n cheapest chat models.
|
||||
"""
|
||||
from litellm.utils import bundled_provider_models
|
||||
|
||||
if custom_llm_provider not in litellm.models_by_provider:
|
||||
return []
|
||||
|
||||
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
|
||||
known_models: Final = bundled_provider_models(custom_llm_provider)
|
||||
ranked_rows: Final = (
|
||||
(route_rank, cost, model)
|
||||
for model, cost in _chat_row_costs(known_models, custom_llm_provider)
|
||||
|
|
|
|||
|
|
@ -8785,6 +8785,12 @@ def _get_valid_models_from_provider_api(
|
|||
return []
|
||||
|
||||
|
||||
def bundled_provider_models(provider: str) -> tuple[str, ...]:
|
||||
bundled: Final[tuple[str, ...]] = tuple(litellm.models_by_provider.get(provider, ()))
|
||||
cost_map: Final = _builtin_cost_map()
|
||||
return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {})))
|
||||
|
||||
|
||||
def get_valid_models(
|
||||
check_provider_endpoint: bool | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
|
|
@ -8850,11 +8856,9 @@ def get_valid_models(
|
|||
)
|
||||
)
|
||||
else:
|
||||
models_for_provider = copy.deepcopy(litellm.models_by_provider.get(provider, []))
|
||||
valid_models.extend(models_for_provider)
|
||||
valid_models.extend(bundled_provider_models(provider))
|
||||
|
||||
cost_map: Final = _builtin_cost_map()
|
||||
return [model for model in valid_models if not is_hidden_as_retired(cost_map.get(model, {}))]
|
||||
return valid_models
|
||||
except Exception as e:
|
||||
verbose_logger.warning("Error getting valid models: %s", e)
|
||||
return [] # NON-Blocking
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import respx
|
|||
|
||||
import litellm
|
||||
from litellm.llms.baseten.chat import BasetenConfig
|
||||
from litellm.utils import get_valid_models
|
||||
from litellm.utils import _model_cache, get_valid_models
|
||||
|
||||
|
||||
class TestBasetenRouting:
|
||||
|
|
@ -92,6 +92,20 @@ class TestBasetenModelDiscovery:
|
|||
litellm.baseten_models.discard(model)
|
||||
litellm.add_known_models({})
|
||||
|
||||
def test_wildcard_drops_a_model_whose_retirement_date_passes_while_running(self, monkeypatch: pytest.MonkeyPatch):
|
||||
model: Final = "baseten/acme/retires-while-running"
|
||||
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "baseten", "deprecation_date": "2999-01-01"})
|
||||
try:
|
||||
litellm.add_known_models({model: litellm.model_cost[model]})
|
||||
assert model in get_valid_models(custom_llm_provider="baseten")
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"})
|
||||
|
||||
assert model not in get_valid_models(custom_llm_provider="baseten")
|
||||
finally:
|
||||
litellm.baseten_models.discard(model)
|
||||
litellm.add_known_models({})
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value, expected",
|
||||
[
|
||||
|
|
@ -156,6 +170,18 @@ class TestBasetenModelDiscovery:
|
|||
|
||||
assert models == ["baseten/brand-new/model"]
|
||||
|
||||
@patch.dict(os.environ, {"BASETEN_API_KEY": "test-key"})
|
||||
def test_live_catalog_keeps_a_model_the_cost_map_dates_as_retired(self, respx_mock: respx.MockRouter) -> None:
|
||||
assert litellm._is_past_deprecation_date(litellm.model_cost["baseten/zai-org/GLM-5"])
|
||||
_model_cache.flush_cache()
|
||||
respx_mock.get("https://inference.baseten.co/v1/models").mock(
|
||||
return_value=httpx.Response(200, json={"data": [{"id": "zai-org/GLM-5"}]})
|
||||
)
|
||||
|
||||
models: Final = get_valid_models(check_provider_endpoint=True, custom_llm_provider="baseten")
|
||||
|
||||
assert models == ["baseten/zai-org/GLM-5"]
|
||||
|
||||
|
||||
class TestBasetenReasoningEffort:
|
||||
def test_reasoning_effort_passthrough_with_drop_params(self):
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import respx
|
|||
import litellm
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.litellm_core_utils.llm_request_utils import pick_cheapest_chat_models_from_llm_provider
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.proxy.auth.model_checks import get_known_models_from_wildcard
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
|
@ -123,6 +124,41 @@ def test_aiand_wildcard_drops_a_model_whose_retirement_date_passes_while_running
|
|||
litellm.add_known_models({})
|
||||
|
||||
|
||||
def test_aiand_models_by_provider_leaves_out_models_retired_at_load():
|
||||
retired: Final = frozenset(
|
||||
name
|
||||
for name, row in litellm.model_cost.items()
|
||||
if row.get("litellm_provider") == "aiand" and litellm._is_past_deprecation_date(row)
|
||||
)
|
||||
|
||||
assert retired
|
||||
assert retired.isdisjoint(litellm.models_by_provider["aiand"])
|
||||
|
||||
|
||||
def test_aiand_wildcard_health_check_skips_a_model_whose_retirement_date_passes_while_running(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
model: Final = "aiand/acme/free-and-retiring"
|
||||
live_row: Final = {
|
||||
"litellm_provider": "aiand",
|
||||
"mode": "chat",
|
||||
"input_cost_per_token": 0.0,
|
||||
"output_cost_per_token": 0.0,
|
||||
"deprecation_date": "2999-01-01",
|
||||
}
|
||||
monkeypatch.setitem(litellm.model_cost, model, live_row)
|
||||
try:
|
||||
litellm.add_known_models({model: live_row})
|
||||
assert model in pick_cheapest_chat_models_from_llm_provider("aiand", n=1)
|
||||
|
||||
monkeypatch.setitem(litellm.model_cost, model, {**live_row, "deprecation_date": "2000-01-01"})
|
||||
|
||||
assert model not in pick_cheapest_chat_models_from_llm_provider("aiand", n=3)
|
||||
finally:
|
||||
litellm.aiand_models.discard(model)
|
||||
litellm.add_known_models({})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("host_zone", ["Etc/GMT+12", "Etc/GMT-14"])
|
||||
def test_aiand_models_retire_on_the_utc_date_whatever_the_host_zone(monkeypatch: pytest.MonkeyPatch, host_zone: str):
|
||||
today: Final = _utc_today()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue