diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index db881aa16c6..d2ff081efb3 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -151,10 +151,12 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): Returns: list[str]: A list of the n cheapest chat models. """ + from litellm.utils import bundled_provider_models + if custom_llm_provider not in litellm.models_by_provider: return [] - known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ()) + known_models: Final = bundled_provider_models(custom_llm_provider) ranked_rows: Final = ( (route_rank, cost, model) for model, cost in _chat_row_costs(known_models, custom_llm_provider) diff --git a/litellm/utils.py b/litellm/utils.py index ea5c7cb0a51..effefc1af06 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8785,6 +8785,12 @@ def _get_valid_models_from_provider_api( return [] +def bundled_provider_models(provider: str) -> tuple[str, ...]: + bundled: Final[tuple[str, ...]] = tuple(litellm.models_by_provider.get(provider, ())) + cost_map: Final = _builtin_cost_map() + return tuple(model for model in bundled if not is_hidden_as_retired(cost_map.get(model, {}))) + + def get_valid_models( check_provider_endpoint: bool | None = None, custom_llm_provider: str | None = None, @@ -8850,11 +8856,9 @@ def get_valid_models( ) ) else: - models_for_provider = copy.deepcopy(litellm.models_by_provider.get(provider, [])) - valid_models.extend(models_for_provider) + valid_models.extend(bundled_provider_models(provider)) - cost_map: Final = _builtin_cost_map() - return [model for model in valid_models if not is_hidden_as_retired(cost_map.get(model, {}))] + return valid_models except Exception as e: verbose_logger.warning("Error getting valid models: %s", e) return [] # NON-Blocking diff --git a/tests/unit/llms/baseten/chat/test_baseten_completions.py b/tests/unit/llms/baseten/chat/test_baseten_completions.py index 4f135a1a106..deb316f81b0 100644 --- a/tests/unit/llms/baseten/chat/test_baseten_completions.py +++ b/tests/unit/llms/baseten/chat/test_baseten_completions.py @@ -8,7 +8,7 @@ import respx import litellm from litellm.llms.baseten.chat import BasetenConfig -from litellm.utils import get_valid_models +from litellm.utils import _model_cache, get_valid_models class TestBasetenRouting: @@ -92,6 +92,20 @@ class TestBasetenModelDiscovery: litellm.baseten_models.discard(model) litellm.add_known_models({}) + def test_wildcard_drops_a_model_whose_retirement_date_passes_while_running(self, monkeypatch: pytest.MonkeyPatch): + model: Final = "baseten/acme/retires-while-running" + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "baseten", "deprecation_date": "2999-01-01"}) + try: + litellm.add_known_models({model: litellm.model_cost[model]}) + assert model in get_valid_models(custom_llm_provider="baseten") + + monkeypatch.setitem(litellm.model_cost, model, {"litellm_provider": "baseten", "deprecation_date": "2000-01-01"}) + + assert model not in get_valid_models(custom_llm_provider="baseten") + finally: + litellm.baseten_models.discard(model) + litellm.add_known_models({}) + @pytest.mark.parametrize( "value, expected", [ @@ -156,6 +170,18 @@ class TestBasetenModelDiscovery: assert models == ["baseten/brand-new/model"] + @patch.dict(os.environ, {"BASETEN_API_KEY": "test-key"}) + def test_live_catalog_keeps_a_model_the_cost_map_dates_as_retired(self, respx_mock: respx.MockRouter) -> None: + assert litellm._is_past_deprecation_date(litellm.model_cost["baseten/zai-org/GLM-5"]) + _model_cache.flush_cache() + respx_mock.get("https://inference.baseten.co/v1/models").mock( + return_value=httpx.Response(200, json={"data": [{"id": "zai-org/GLM-5"}]}) + ) + + models: Final = get_valid_models(check_provider_endpoint=True, custom_llm_provider="baseten") + + assert models == ["baseten/zai-org/GLM-5"] + class TestBasetenReasoningEffort: def test_reasoning_effort_passthrough_with_drop_params(self): diff --git a/tests/unit/llms/openai_like/test_aiand_provider.py b/tests/unit/llms/openai_like/test_aiand_provider.py index 2defec92b4e..bd517a46c0e 100644 --- a/tests/unit/llms/openai_like/test_aiand_provider.py +++ b/tests/unit/llms/openai_like/test_aiand_provider.py @@ -10,6 +10,7 @@ import respx import litellm from litellm.caching.llm_caching_handler import LLMClientCache from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.litellm_core_utils.llm_request_utils import pick_cheapest_chat_models_from_llm_provider from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.proxy.auth.model_checks import get_known_models_from_wildcard from litellm.utils import ProviderConfigManager @@ -123,6 +124,41 @@ def test_aiand_wildcard_drops_a_model_whose_retirement_date_passes_while_running litellm.add_known_models({}) +def test_aiand_models_by_provider_leaves_out_models_retired_at_load(): + retired: Final = frozenset( + name + for name, row in litellm.model_cost.items() + if row.get("litellm_provider") == "aiand" and litellm._is_past_deprecation_date(row) + ) + + assert retired + assert retired.isdisjoint(litellm.models_by_provider["aiand"]) + + +def test_aiand_wildcard_health_check_skips_a_model_whose_retirement_date_passes_while_running( + monkeypatch: pytest.MonkeyPatch, +): + model: Final = "aiand/acme/free-and-retiring" + live_row: Final = { + "litellm_provider": "aiand", + "mode": "chat", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "deprecation_date": "2999-01-01", + } + monkeypatch.setitem(litellm.model_cost, model, live_row) + try: + litellm.add_known_models({model: live_row}) + assert model in pick_cheapest_chat_models_from_llm_provider("aiand", n=1) + + monkeypatch.setitem(litellm.model_cost, model, {**live_row, "deprecation_date": "2000-01-01"}) + + assert model not in pick_cheapest_chat_models_from_llm_provider("aiand", n=3) + finally: + litellm.aiand_models.discard(model) + litellm.add_known_models({}) + + @pytest.mark.parametrize("host_zone", ["Etc/GMT+12", "Etc/GMT-14"]) def test_aiand_models_retire_on_the_utc_date_whatever_the_host_zone(monkeypatch: pytest.MonkeyPatch, host_zone: str): today: Final = _utc_today()