fix(health): keep wildcard health picks that only route through a saved custom_llm_provider

Rank picks routed to the checked provider first and ids routed to no provider
second, each by cost, and skip only ids routed to a different provider, so a
wildcard saved with custom_llm_provider still sends its health request
This commit is contained in:
mateo-berri 2026-10-28 21:55:42 -07:00
parent e598104c7a
commit 20880a6731
3 changed files with 39 additions and 6 deletions

View file

@ -1,5 +1,4 @@
from collections.abc import Iterable, Iterator, Mapping
from itertools import islice
from typing import Final, cast
import litellm
@ -156,8 +155,12 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
return []
known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
cheapest_first: Final = sorted(_chat_row_costs(known_models, custom_llm_provider), key=lambda row: row[1])
return list(islice((model for model, _ in cheapest_first if _routes_to(model, custom_llm_provider)), n))
ranked_rows: Final = (
(route_rank, cost, model)
for model, cost in _chat_row_costs(known_models, custom_llm_provider)
if (route_rank := _route_rank(model, custom_llm_provider)) is not None
)
return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
@ -171,9 +174,13 @@ def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator
yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
def _routes_to(model: str, custom_llm_provider: str) -> bool:
def _route_rank(model: str, custom_llm_provider: str) -> int | None:
routed_provider: Final = inferred_provider(model)
return routed_provider is not None and routed_provider == inferred_provider(f"{custom_llm_provider}/{model}")
if routed_provider is None:
return 1
if routed_provider == inferred_provider(f"{custom_llm_provider}/{model}"):
return 0
return None
def get_proxy_server_request_headers(litellm_params: dict | None) -> dict:

View file

@ -1620,3 +1620,28 @@ async def test_ai21_wildcard_health_check_calls_ai21(
assert ai21_route.called
assert not other_hosts.called
async def test_wildcard_health_check_with_pinned_provider_calls_a_model_without_a_routing_prefix(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
monkeypatch.setitem(
litellm.model_cost,
"fake-together-chat",
{"input_cost_per_token": 1e-9, "output_cost_per_token": 1e-9, "mode": "chat", "litellm_provider": "together_ai"},
)
monkeypatch.setitem(litellm.models_by_provider, "together_ai", ("fake-together-chat",))
litellm.in_memory_llm_clients_cache.flush_cache()
together_route: Final = respx_mock.route(host="api.together.ai").respond(json=dict(CHAT_COMPLETION_BODY))
other_hosts: Final = respx_mock.route().respond(json=dict(CHAT_COMPLETION_BODY))
result: Final = await ahealth_check(
model_params={"model": "together_ai/*", "custom_llm_provider": "together_ai", "api_key": "health-together-key"},
mode="chat",
)
assert "error" not in result, result
assert json.loads(together_route.calls.last.request.content)["model"] == "fake-together-chat"
assert not other_hosts.called

View file

@ -156,11 +156,12 @@ def deepseek_rows_whose_cheapest_ids_route_elsewhere(monkeypatch: pytest.MonkeyP
monkeypatch.setitem(litellm.models_by_provider, PICKER_PROVIDER, tuple(rows))
def test_pick_cheapest_chat_models_offers_only_ids_the_router_sends_to_the_provider(
def test_pick_cheapest_chat_models_prefers_provider_routed_ids_and_skips_ids_routed_elsewhere(
deepseek_rows_whose_cheapest_ids_route_elsewhere: None,
) -> None:
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=10) == [
"deepseek/fake-cheap-chat",
"deepseek/fake-pricey-chat",
"fake-free-chat-without-provider",
]
assert pick_cheapest_chat_models_from_llm_provider(PICKER_PROVIDER, n=1) == ["deepseek/fake-cheap-chat"]