diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index 8ce0be5e436..db881aa16c6 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -1,5 +1,4 @@ from collections.abc import Iterable, Iterator, Mapping -from itertools import islice from typing import Final, cast import litellm @@ -156,8 +155,12 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1): return [] known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ()) - cheapest_first: Final = sorted(_chat_row_costs(known_models, custom_llm_provider), key=lambda row: row[1]) - return list(islice((model for model, _ in cheapest_first if _routes_to(model, custom_llm_provider)), n)) + ranked_rows: Final = ( + (route_rank, cost, model) + for model, cost in _chat_row_costs(known_models, custom_llm_provider) + if (route_rank := _route_rank(model, custom_llm_provider)) is not None + ) + return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]] def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]: @@ -171,9 +174,13 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0) -def _routes_to(model: str, custom_llm_provider: str) -> bool: +def _route_rank(model: str, custom_llm_provider: str) -> int | None: routed_provider: Final = inferred_provider(model) - return routed_provider is not None and routed_provider == inferred_provider(f"{custom_llm_provider}/{model}") + if routed_provider is None: + return 1 + if routed_provider == inferred_provider(f"{custom_llm_provider}/{model}"): + return 0 + return None def get_proxy_server_request_headers(litellm_params: dict | None) -> dict: diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index d1708d4c961..44b36a355c6 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1620,3 +1620,28 @@ async def test_ai21_wildcard_health_check_calls_ai21( assert ai21_route.called assert not other_hosts.called + +async def test_wildcard_health_check_with_pinned_provider_calls_a_model_without_a_routing_prefix( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setitem( + litellm.model_cost, + "fake-together-chat", + {"input_cost_per_token": 1e-9, "output_cost_per_token": 1e-9, "mode": "chat", "litellm_provider": "together_ai"}, + ) + monkeypatch.setitem(litellm.models_by_provider, "together_ai", ("fake-together-chat",)) + litellm.in_memory_llm_clients_cache.flush_cache() + together_route: Final = respx_mock.route(host="api.together.ai").respond(json=dict(CHAT_COMPLETION_BODY)) + other_hosts: Final = respx_mock.route().respond(json=dict(CHAT_COMPLETION_BODY)) + + result: Final = await ahealth_check( + model_params={"model": "together_ai/*", "custom_llm_provider": "together_ai", "api_key": "health-together-key"}, + mode="chat", + ) + + assert "error" not in result, result + assert json.loads(together_route.calls.last.request.content)["model"] == "fake-together-chat" + assert not other_hosts.called + diff --git a/tests/unit/litellm_core_utils/test_llm_request_utils.py b/tests/unit/litellm_core_utils/test_llm_request_utils.py index 20bbfa7ce82..0a2bd5cec1b 100644 --- a/tests/unit/litellm_core_utils/test_llm_request_utils.py +++ b/tests/unit/litellm_core_utils/test_llm_request_utils.py @@ -156,11 +156,12 @@ def deepseek_rows_whose_cheapest_ids_route_elsewhere(monkeypatch: pytest.MonkeyP monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(rows)) -def test_pick_cheapest_chat_models_offers_only_ids_the_router_sends_to_the_provider( +def test_pick_cheapest_chat_models_prefers_provider_routed_ids_and_skips_ids_routed_elsewhere( deepseek_rows_whose_cheapest_ids_route_elsewhere: None, ) -> None: assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=10) == [ "deepseek/fake-cheap-chat", "deepseek/fake-pricey-chat", + "fake-free-chat-without-provider", ] assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=1) == ["deepseek/fake-cheap-chat"]